siw.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2
  3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
  4/* Copyright (c) 2008-2019, IBM Corporation */
  5
  6#ifndef _SIW_H
  7#define _SIW_H
  8
  9#include <rdma/ib_verbs.h>
 10#include <rdma/restrack.h>
 11#include <linux/socket.h>
 12#include <linux/skbuff.h>
 13#include <linux/crc32.h>
 14#include <linux/crc32c.h>
 15#include <linux/unaligned.h>
 16
 17#include <rdma/siw-abi.h>
 18#include "iwarp.h"
 19
 20#define SIW_VENDOR_ID 0x626d74 /* ascii 'bmt' for now */
 21#define SIW_VENDORT_PART_ID 0
 22#define SIW_MAX_QP (1024 * 100)
 23#define SIW_MAX_QP_WR (1024 * 32)
 24#define SIW_MAX_ORD_QP 128
 25#define SIW_MAX_IRD_QP 128
 26#define SIW_MAX_SGE_PBL 256 /* max num sge's for PBL */
 27#define SIW_MAX_SGE_RD 1 /* iwarp limitation. we could relax */
 28#define SIW_MAX_CQ (1024 * 100)
 29#define SIW_MAX_CQE (SIW_MAX_QP_WR * 100)
 30#define SIW_MAX_MR (SIW_MAX_QP * 10)
 31#define SIW_MAX_PD SIW_MAX_QP
 32#define SIW_MAX_MW 0 /* to be set if MW's are supported */
 33#define SIW_MAX_SRQ SIW_MAX_QP
 34#define SIW_MAX_SRQ_WR (SIW_MAX_QP_WR * 10)
 35#define SIW_MAX_CONTEXT SIW_MAX_PD
 36
 37/* Min number of bytes for using zero copy transmit */
 38#define SENDPAGE_THRESH PAGE_SIZE
 39
 40/* Maximum number of frames which can be send in one SQ processing */
 41#define SQ_USER_MAXBURST 100
 42
 43/* Maximum number of consecutive IRQ elements which get served
 44 * if SQ has pending work. Prevents starving local SQ processing
 45 * by serving peer Read Requests.
 46 */
 47#define SIW_IRQ_MAXBURST_SQ_ACTIVE 4
 48
 49/* There is always only a port 1 per siw device */
 50#define SIW_PORT 1
 51
 52struct siw_dev_cap {
 53	int max_qp;
 54	int max_qp_wr;
 55	int max_ord; /* max. outbound read queue depth */
 56	int max_ird; /* max. inbound read queue depth */
 57	int max_sge;
 58	int max_sge_rd;
 59	int max_cq;
 60	int max_cqe;
 61	int max_mr;
 62	int max_pd;
 63	int max_mw;
 64	int max_srq;
 65	int max_srq_wr;
 66	int max_srq_sge;
 67};
 68
 69struct siw_pd {
 70	struct ib_pd base_pd;
 71};
 72
 73struct siw_device {
 74	struct ib_device base_dev;
 75	struct siw_dev_cap attrs;
 76
 77	u32 vendor_part_id;
 78	int numa_node;
 79	char raw_gid[ETH_ALEN];
 80
 81	spinlock_t lock;
 82
 83	struct xarray qp_xa;
 84	struct xarray mem_xa;
 85
 86	struct list_head cep_list;
 87	struct list_head qp_list;
 88
 89	/* active objects statistics to enforce limits */
 90	atomic_t num_qp;
 91	atomic_t num_cq;
 92	atomic_t num_pd;
 93	atomic_t num_mr;
 94	atomic_t num_srq;
 95	atomic_t num_ctx;
 96};
 97
 98struct siw_ucontext {
 99	struct ib_ucontext base_ucontext;
100	struct siw_device *sdev;
101};
102
103/*
104 * The RDMA core does not define LOCAL_READ access, which is always
105 * enabled implictely.
106 */
107#define IWARP_ACCESS_MASK					\
108	(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE	|	\
109	 IB_ACCESS_REMOTE_READ)
110
111/*
112 * siw presentation of user memory registered as source
113 * or target of RDMA operations.
114 */
115
116struct siw_page_chunk {
117	struct page **plist;
118};
119
120struct siw_umem {
121	struct ib_umem *base_mem;
122	struct siw_page_chunk *page_chunk;
123	int num_pages;
124	u64 fp_addr; /* First page base address */
125};
126
127struct siw_pble {
128	dma_addr_t addr; /* Address of assigned buffer */
129	unsigned int size; /* Size of this entry */
130	unsigned long pbl_off; /* Total offset from start of PBL */
131};
132
133struct siw_pbl {
134	unsigned int num_buf;
135	unsigned int max_buf;
136	struct siw_pble pbe[] __counted_by(max_buf);
137};
138
139/*
140 * Generic memory representation for registered siw memory.
141 * Memory lookup always via higher 24 bit of STag (STag index).
142 */
143struct siw_mem {
144	struct siw_device *sdev;
145	struct kref ref;
146	u64 va; /* VA of memory */
147	u64 len; /* length of the memory buffer in bytes */
148	u32 stag; /* iWarp memory access steering tag */
149	u8 stag_valid; /* VALID or INVALID */
150	u8 is_pbl; /* PBL or user space mem */
151	u8 is_mw; /* Memory Region or Memory Window */
152	enum ib_access_flags perms; /* local/remote READ & WRITE */
153	union {
154		struct siw_umem *umem;
155		struct siw_pbl *pbl;
156		void *mem_obj;
157	};
158	struct ib_pd *pd;
159};
160
161struct siw_mr {
162	struct ib_mr base_mr;
163	struct siw_mem *mem;
164	struct rcu_head rcu;
165};
166
167/*
168 * Error codes for local or remote
169 * access to registered memory
170 */
171enum siw_access_state {
172	E_ACCESS_OK,
173	E_STAG_INVALID,
174	E_BASE_BOUNDS,
175	E_ACCESS_PERM,
176	E_PD_MISMATCH
177};
178
179enum siw_wr_state {
180	SIW_WR_IDLE,
181	SIW_WR_QUEUED, /* processing has not started yet */
182	SIW_WR_INPROGRESS /* initiated processing of the WR */
183};
184
185/* The WQE currently being processed (RX or TX) */
186struct siw_wqe {
187	/* Copy of applications SQE or RQE */
188	union {
189		struct siw_sqe sqe;
190		struct siw_rqe rqe;
191	};
192	struct siw_mem *mem[SIW_MAX_SGE]; /* per sge's resolved mem */
193	enum siw_wr_state wr_status;
194	enum siw_wc_status wc_status;
195	u32 bytes; /* total bytes to process */
196	u32 processed; /* bytes processed */
197};
198
199struct siw_cq {
200	struct ib_cq base_cq;
201	spinlock_t lock;
202	struct siw_cq_ctrl *notify;
203	struct siw_cqe *queue;
204	u32 cq_put;
205	u32 cq_get;
206	u32 num_cqe;
207	struct rdma_user_mmap_entry *cq_entry; /* mmap info for CQE array */
208	u32 id; /* For debugging only */
209};
210
211enum siw_qp_state {
212	SIW_QP_STATE_IDLE,
213	SIW_QP_STATE_RTR,
214	SIW_QP_STATE_RTS,
215	SIW_QP_STATE_CLOSING,
216	SIW_QP_STATE_TERMINATE,
217	SIW_QP_STATE_ERROR,
218	SIW_QP_STATE_COUNT
219};
220
221enum siw_qp_flags {
222	SIW_RDMA_BIND_ENABLED = (1 << 0),
223	SIW_RDMA_WRITE_ENABLED = (1 << 1),
224	SIW_RDMA_READ_ENABLED = (1 << 2),
225	SIW_SIGNAL_ALL_WR = (1 << 3),
226	SIW_MPA_CRC = (1 << 4),
227	SIW_QP_IN_DESTROY = (1 << 5)
228};
229
230enum siw_qp_attr_mask {
231	SIW_QP_ATTR_STATE = (1 << 0),
232	SIW_QP_ATTR_ACCESS_FLAGS = (1 << 1),
233	SIW_QP_ATTR_LLP_HANDLE = (1 << 2),
234	SIW_QP_ATTR_ORD = (1 << 3),
235	SIW_QP_ATTR_IRD = (1 << 4),
236	SIW_QP_ATTR_SQ_SIZE = (1 << 5),
237	SIW_QP_ATTR_RQ_SIZE = (1 << 6),
238	SIW_QP_ATTR_MPA = (1 << 7)
239};
240
241struct siw_srq {
242	struct ib_srq base_srq;
243	spinlock_t lock;
244	u32 max_sge;
245	u32 limit; /* low watermark for async event */
246	struct siw_rqe *recvq;
247	u32 rq_put;
248	u32 rq_get;
249	u32 num_rqe; /* max # of wqe's allowed */
250	struct rdma_user_mmap_entry *srq_entry; /* mmap info for SRQ array */
251	bool armed:1; /* inform user if limit hit */
252	bool is_kernel_res:1; /* true if kernel client */
253};
254
255struct siw_qp_attrs {
256	enum siw_qp_state state;
257	u32 sq_size;
258	u32 rq_size;
259	u32 orq_size;
260	u32 irq_size;
261	u32 sq_max_sges;
262	u32 rq_max_sges;
263	enum siw_qp_flags flags;
264
265	struct socket *sk;
266};
267
268enum siw_tx_ctx {
269	SIW_SEND_HDR, /* start or continue sending HDR */
270	SIW_SEND_DATA, /* start or continue sending DDP payload */
271	SIW_SEND_TRAILER, /* start or continue sending TRAILER */
272	SIW_SEND_SHORT_FPDU/* send whole FPDU hdr|data|trailer at once */
273};
274
275enum siw_rx_state {
276	SIW_GET_HDR, /* await new hdr or within hdr */
277	SIW_GET_DATA_START, /* start of inbound DDP payload */
278	SIW_GET_DATA_MORE, /* continuation of (misaligned) DDP payload */
279	SIW_GET_TRAILER/* await new trailer or within trailer */
280};
281
282struct siw_rx_stream {
283	struct sk_buff *skb;
284	int skb_new; /* pending unread bytes in skb */
285	int skb_offset; /* offset in skb */
286	int skb_copied; /* processed bytes in skb */
287
288	enum siw_rx_state state;
289
290	union iwarp_hdr hdr;
291	struct mpa_trailer trailer;
292	u32 mpa_crc;
293	bool mpa_crc_enabled;
294
295	/*
296	 * For each FPDU, main RX loop runs through 3 stages:
297	 * Receiving protocol headers, placing DDP payload and receiving
298	 * trailer information (CRC + possibly padding).
299	 * Next two variables keep state on receive status of the
300	 * current FPDU part (hdr, data, trailer).
301	 */
302	int fpdu_part_rcvd; /* bytes in pkt part copied */
303	int fpdu_part_rem; /* bytes in pkt part not seen */
304
305	/*
306	 * Next expected DDP MSN for each QN +
307	 * expected steering tag +
308	 * expected DDP tagget offset (all HBO)
309	 */
310	u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT];
311	u32 ddp_stag;
312	u64 ddp_to;
313	u32 inval_stag; /* Stag to be invalidated */
314
315	u8 rx_suspend : 1;
316	u8 pad : 2; /* # of pad bytes expected */
317	u8 rdmap_op : 4; /* opcode of current frame */
318};
319
320struct siw_rx_fpdu {
321	/*
322	 * Local destination memory of inbound RDMA operation.
323	 * Valid, according to wqe->wr_status
324	 */
325	struct siw_wqe wqe_active;
326
327	unsigned int pbl_idx; /* Index into current PBL */
328	unsigned int sge_idx; /* current sge in rx */
329	unsigned int sge_off; /* already rcvd in curr. sge */
330
331	char first_ddp_seg; /* this is the first DDP seg */
332	char more_ddp_segs; /* more DDP segs expected */
333	u8 prev_rdmap_op : 4; /* opcode of prev frame */
334};
335
336/*
337 * Shorthands for short packets w/o payload
338 * to be transmitted more efficient.
339 */
340struct siw_send_pkt {
341	struct iwarp_send send;
342	__be32 crc;
343};
344
345struct siw_write_pkt {
346	struct iwarp_rdma_write write;
347	__be32 crc;
348};
349
350struct siw_rreq_pkt {
351	struct iwarp_rdma_rreq rreq;
352	__be32 crc;
353};
354
355struct siw_rresp_pkt {
356	struct iwarp_rdma_rresp rresp;
357	__be32 crc;
358};
359
360struct siw_iwarp_tx {
361	union {
362		union iwarp_hdr hdr;
363
364		/* Generic part of FPDU header */
365		struct iwarp_ctrl ctrl;
366		struct iwarp_ctrl_untagged c_untagged;
367		struct iwarp_ctrl_tagged c_tagged;
368
369		/* FPDU headers */
370		struct iwarp_rdma_write rwrite;
371		struct iwarp_rdma_rreq rreq;
372		struct iwarp_rdma_rresp rresp;
373		struct iwarp_terminate terminate;
374		struct iwarp_send send;
375		struct iwarp_send_inv send_inv;
376
377		/* complete short FPDUs */
378		struct siw_send_pkt send_pkt;
379		struct siw_write_pkt write_pkt;
380		struct siw_rreq_pkt rreq_pkt;
381		struct siw_rresp_pkt rresp_pkt;
382	} pkt;
383
384	struct mpa_trailer trailer;
385	/* DDP MSN for untagged messages */
386	u32 ddp_msn[RDMAP_UNTAGGED_QN_COUNT];
387
388	enum siw_tx_ctx state;
389	u16 ctrl_len; /* ddp+rdmap hdr */
390	u16 ctrl_sent;
391	int burst;
392	int bytes_unsent; /* ddp payload bytes */
393
394	u32 mpa_crc;
395	bool mpa_crc_enabled;
396
397	u8 do_crc : 1; /* do crc for segment */
398	u8 use_sendpage : 1; /* send w/o copy */
399	u8 tx_suspend : 1; /* stop sending DDP segs. */
400	u8 pad : 2; /* # pad in current fpdu */
401	u8 orq_fence : 1; /* ORQ full or Send fenced */
402	u8 in_syscall : 1; /* TX out of user context */
403	u8 zcopy_tx : 1; /* Use TCP_SENDPAGE if possible */
404	u8 gso_seg_limit; /* Maximum segments for GSO, 0 = unbound */
405
406	u16 fpdu_len; /* len of FPDU to tx */
407	unsigned int tcp_seglen; /* remaining tcp seg space */
408
409	struct siw_wqe wqe_active;
410
411	int pbl_idx; /* Index into current PBL */
412	int sge_idx; /* current sge in tx */
413	u32 sge_off; /* already sent in curr. sge */
414};
415
416struct siw_qp {
417	struct ib_qp base_qp;
418	struct siw_device *sdev;
419	int tx_cpu;
420	struct kref ref;
421	struct completion qp_free;
422	struct list_head devq;
423	struct siw_qp_attrs attrs;
424
425	struct siw_cep *cep;
426	struct rw_semaphore state_lock;
427
428	struct ib_pd *pd;
429	struct siw_cq *scq;
430	struct siw_cq *rcq;
431	struct siw_srq *srq;
432
433	struct siw_iwarp_tx tx_ctx; /* Transmit context */
434	spinlock_t sq_lock;
435	struct siw_sqe *sendq; /* send queue element array */
436	uint32_t sq_get; /* consumer index into sq array */
437	uint32_t sq_put; /* kernel prod. index into sq array */
438	struct llist_node tx_list;
439
440	struct siw_sqe *orq; /* outbound read queue element array */
441	spinlock_t orq_lock;
442	uint32_t orq_get; /* consumer index into orq array */
443	uint32_t orq_put; /* shared producer index for ORQ */
444
445	struct siw_rx_stream rx_stream;
446	struct siw_rx_fpdu *rx_fpdu;
447	struct siw_rx_fpdu rx_tagged;
448	struct siw_rx_fpdu rx_untagged;
449	spinlock_t rq_lock;
450	struct siw_rqe *recvq; /* recv queue element array */
451	uint32_t rq_get; /* consumer index into rq array */
452	uint32_t rq_put; /* kernel prod. index into rq array */
453
454	struct siw_sqe *irq; /* inbound read queue element array */
455	uint32_t irq_get; /* consumer index into irq array */
456	uint32_t irq_put; /* producer index into irq array */
457	int irq_burst;
458
459	struct { /* information to be carried in TERMINATE pkt, if valid */
460		u8 valid;
461		u8 in_tx;
462		u8 layer : 4, etype : 4;
463		u8 ecode;
464	} term_info;
465	struct rdma_user_mmap_entry *sq_entry; /* mmap info for SQE array */
466	struct rdma_user_mmap_entry *rq_entry; /* mmap info for RQE array */
467};
468
469/* helper macros */
470#define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream)
471#define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx)
472#define tx_wqe(qp) (&(qp)->tx_ctx.wqe_active)
473#define rx_wqe(rctx) (&(rctx)->wqe_active)
474#define rx_mem(rctx) ((rctx)->wqe_active.mem[0])
475#define tx_type(wqe) ((wqe)->sqe.opcode)
476#define rx_type(wqe) ((wqe)->rqe.opcode)
477#define tx_flags(wqe) ((wqe)->sqe.flags)
478
479struct iwarp_msg_info {
480	int hdr_len;
481	struct iwarp_ctrl ctrl;
482	int (*rx_data)(struct siw_qp *qp);
483};
484
485struct siw_user_mmap_entry {
486	struct rdma_user_mmap_entry rdma_entry;
487	void *address;
488};
489
490/* Global siw parameters. Currently set in siw_main.c */
491extern const bool zcopy_tx;
492extern const bool try_gso;
493extern const bool loopback_enabled;
494extern const bool mpa_crc_required;
495extern const bool mpa_crc_strict;
496extern const bool siw_tcp_nagle;
497extern u_char mpa_version;
498extern const bool peer_to_peer;
499extern struct task_struct *siw_tx_thread[];
500
501extern struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1];
502
503/* QP general functions */
504int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attr,
505		  enum siw_qp_attr_mask mask);
506int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl);
507void siw_qp_llp_close(struct siw_qp *qp);
508void siw_qp_cm_drop(struct siw_qp *qp, int schedule);
509void siw_send_terminate(struct siw_qp *qp);
510
511void siw_qp_get_ref(struct ib_qp *qp);
512void siw_qp_put_ref(struct ib_qp *qp);
513int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp);
514void siw_free_qp(struct kref *ref);
515
516void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer,
517			u8 etype, u8 ecode, int in_tx);
518enum ddp_ecode siw_tagged_error(enum siw_access_state state);
519enum rdmap_ecode siw_rdmap_error(enum siw_access_state state);
520
521void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe);
522int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
523		     enum siw_wc_status status);
524int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
525		     u32 inval_stag, enum siw_wc_status status);
526void siw_qp_llp_data_ready(struct sock *sk);
527void siw_qp_llp_write_space(struct sock *sk);
528
529/* QP TX path functions */
530int siw_create_tx_threads(void);
531void siw_stop_tx_threads(void);
532int siw_run_sq(void *arg);
533int siw_qp_sq_process(struct siw_qp *qp);
534int siw_sq_start(struct siw_qp *qp);
535int siw_activate_tx(struct siw_qp *qp);
536int siw_get_tx_cpu(struct siw_device *sdev);
537void siw_put_tx_cpu(int cpu);
538
539/* QP RX path functions */
540int siw_proc_send(struct siw_qp *qp);
541int siw_proc_rreq(struct siw_qp *qp);
542int siw_proc_rresp(struct siw_qp *qp);
543int siw_proc_write(struct siw_qp *qp);
544int siw_proc_terminate(struct siw_qp *qp);
545
546int siw_tcp_rx_data(read_descriptor_t *rd_desc, struct sk_buff *skb,
547		    unsigned int off, size_t len);
548
549static inline void set_rx_fpdu_context(struct siw_qp *qp, u8 opcode)
550{
551	if (opcode == RDMAP_RDMA_WRITE || opcode == RDMAP_RDMA_READ_RESP)
552		qp->rx_fpdu = &qp->rx_tagged;
553	else
554		qp->rx_fpdu = &qp->rx_untagged;
555
556	qp->rx_stream.rdmap_op = opcode;
557}
558
559static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx)
560{
561	return container_of(base_ctx, struct siw_ucontext, base_ucontext);
562}
563
564static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp)
565{
566	return container_of(base_qp, struct siw_qp, base_qp);
567}
568
569static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq)
570{
571	return container_of(base_cq, struct siw_cq, base_cq);
572}
573
574static inline struct siw_srq *to_siw_srq(struct ib_srq *base_srq)
575{
576	return container_of(base_srq, struct siw_srq, base_srq);
577}
578
579static inline struct siw_device *to_siw_dev(struct ib_device *base_dev)
580{
581	return container_of(base_dev, struct siw_device, base_dev);
582}
583
584static inline struct siw_mr *to_siw_mr(struct ib_mr *base_mr)
585{
586	return container_of(base_mr, struct siw_mr, base_mr);
587}
588
589static inline struct siw_user_mmap_entry *
590to_siw_mmap_entry(struct rdma_user_mmap_entry *rdma_mmap)
591{
592	return container_of(rdma_mmap, struct siw_user_mmap_entry, rdma_entry);
593}
594
595static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id)
596{
597	struct siw_qp *qp;
598
599	rcu_read_lock();
600	qp = xa_load(&sdev->qp_xa, id);
601	if (likely(qp && kref_get_unless_zero(&qp->ref))) {
602		rcu_read_unlock();
603		return qp;
604	}
605	rcu_read_unlock();
606	return NULL;
607}
608
609static inline u32 qp_id(struct siw_qp *qp)
610{
611	return qp->base_qp.qp_num;
612}
613
614static inline void siw_qp_get(struct siw_qp *qp)
615{
616	kref_get(&qp->ref);
617}
618
619static inline void siw_qp_put(struct siw_qp *qp)
620{
621	kref_put(&qp->ref, siw_free_qp);
622}
623
624static inline int siw_sq_empty(struct siw_qp *qp)
625{
626	struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
627
628	return READ_ONCE(sqe->flags) == 0;
629}
630
631static inline struct siw_sqe *sq_get_next(struct siw_qp *qp)
632{
633	struct siw_sqe *sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
634
635	if (READ_ONCE(sqe->flags) & SIW_WQE_VALID)
636		return sqe;
637
638	return NULL;
639}
640
641static inline struct siw_sqe *orq_get_current(struct siw_qp *qp)
642{
643	return &qp->orq[qp->orq_get % qp->attrs.orq_size];
644}
645
646static inline struct siw_sqe *orq_get_free(struct siw_qp *qp)
647{
648	struct siw_sqe *orq_e = &qp->orq[qp->orq_put % qp->attrs.orq_size];
649
650	if (READ_ONCE(orq_e->flags) == 0)
651		return orq_e;
652
653	return NULL;
654}
655
656static inline int siw_orq_empty(struct siw_qp *qp)
657{
658	return orq_get_current(qp)->flags == 0 ? 1 : 0;
659}
660
661static inline struct siw_sqe *irq_alloc_free(struct siw_qp *qp)
662{
663	struct siw_sqe *irq_e = &qp->irq[qp->irq_put % qp->attrs.irq_size];
664
665	if (READ_ONCE(irq_e->flags) == 0) {
666		qp->irq_put++;
667		return irq_e;
668	}
669	return NULL;
670}
671
672static inline void siw_crc_init(u32 *crc)
673{
674	*crc = ~0;
675}
676
677static inline void siw_crc_update(u32 *crc, const void *data, size_t len)
678{
679	*crc = crc32c(*crc, data, len);
680}
681
682static inline void siw_crc_final(u32 *crc, u8 out[4])
683{
684	put_unaligned_le32(~*crc, out);
685}
686
687static inline void siw_crc_oneshot(const void *data, size_t len, u8 out[4])
688{
689	u32 crc;
690
691	siw_crc_init(&crc);
692	siw_crc_update(&crc, data, len);
693	return siw_crc_final(&crc, out);
694}
695
696static inline void siw_crc_skb(struct siw_rx_stream *srx, unsigned int len)
697{
698	srx->mpa_crc = skb_crc32c(srx->skb, srx->skb_offset, len, srx->mpa_crc);
699}
700
701#define siw_dbg(ibdev, fmt, ...)                                               \
702	ibdev_dbg(ibdev, "%s: " fmt, __func__, ##__VA_ARGS__)
703
704#define siw_dbg_qp(qp, fmt, ...)                                               \
705	ibdev_dbg(&qp->sdev->base_dev, "QP[%u] %s: " fmt, qp_id(qp), __func__, \
706		  ##__VA_ARGS__)
707
708#define siw_dbg_cq(cq, fmt, ...)                                               \
709	ibdev_dbg(cq->base_cq.device, "CQ[%u] %s: " fmt, cq->id, __func__,     \
710		  ##__VA_ARGS__)
711
712#define siw_dbg_pd(pd, fmt, ...)                                               \
713	ibdev_dbg(pd->device, "PD[%u] %s: " fmt, pd->res.id, __func__,         \
714		  ##__VA_ARGS__)
715
716#define siw_dbg_mem(mem, fmt, ...)                                             \
717	ibdev_dbg(&mem->sdev->base_dev,                                        \
718		  "MEM[0x%08x] %s: " fmt, mem->stag, __func__, ##__VA_ARGS__)
719
720#define siw_dbg_cep(cep, fmt, ...)                                             \
721	ibdev_dbg(&cep->sdev->base_dev, "CEP[0x%p] %s: " fmt,                 \
722		  cep, __func__, ##__VA_ARGS__)
723
724void siw_cq_flush(struct siw_cq *cq);
725void siw_sq_flush(struct siw_qp *qp);
726void siw_rq_flush(struct siw_qp *qp);
727int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc);
728
729#endif