svc_rdma.h

  1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
  2/*
  3 * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved.
  4 *
  5 * This software is available to you under a choice of one of two
  6 * licenses.  You may choose to be licensed under the terms of the GNU
  7 * General Public License (GPL) Version 2, available from the file
  8 * COPYING in the main directory of this source tree, or the BSD-type
  9 * license below:
 10 *
 11 * Redistribution and use in source and binary forms, with or without
 12 * modification, are permitted provided that the following conditions
 13 * are met:
 14 *
 15 *      Redistributions of source code must retain the above copyright
 16 *      notice, this list of conditions and the following disclaimer.
 17 *
 18 *      Redistributions in binary form must reproduce the above
 19 *      copyright notice, this list of conditions and the following
 20 *      disclaimer in the documentation and/or other materials provided
 21 *      with the distribution.
 22 *
 23 *      Neither the name of the Network Appliance, Inc. nor the names of
 24 *      its contributors may be used to endorse or promote products
 25 *      derived from this software without specific prior written
 26 *      permission.
 27 *
 28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 29 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 30 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 31 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 32 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 33 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 34 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 38 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 39 *
 40 * Author: Tom Tucker <tom@opengridcomputing.com>
 41 */
 42
 43#ifndef SVC_RDMA_H
 44#define SVC_RDMA_H
 45#include <linux/llist.h>
 46#include <linux/sunrpc/xdr.h>
 47#include <linux/sunrpc/svcsock.h>
 48#include <linux/sunrpc/rpc_rdma.h>
 49#include <linux/sunrpc/rpc_rdma_cid.h>
 50#include <linux/sunrpc/svc_rdma_pcl.h>
 51#include <linux/sunrpc/rdma_rn.h>
 52
 53#include <linux/percpu_counter.h>
 54#include <rdma/ib_verbs.h>
 55#include <rdma/rdma_cm.h>
 56
 57/* Default and maximum inline threshold sizes */
 58enum {
 59	RPCRDMA_PULLUP_THRESH = RPCRDMA_V1_DEF_INLINE_SIZE >> 1,
 60	RPCRDMA_DEF_INLINE_THRESH = 4096,
 61	RPCRDMA_MAX_INLINE_THRESH = 65536
 62};
 63
 64/* RPC/RDMA parameters and stats */
 65extern unsigned int svcrdma_ord;
 66extern unsigned int svcrdma_max_requests;
 67extern unsigned int svcrdma_max_bc_requests;
 68extern unsigned int svcrdma_max_req_size;
 69extern struct workqueue_struct *svcrdma_wq;
 70
 71extern struct percpu_counter svcrdma_stat_read;
 72extern struct percpu_counter svcrdma_stat_recv;
 73extern struct percpu_counter svcrdma_stat_sq_starve;
 74extern struct percpu_counter svcrdma_stat_write;
 75
 76struct svcxprt_rdma {
 77	struct svc_xprt      sc_xprt;		/* SVC transport structure */
 78	struct rdma_cm_id    *sc_cm_id;		/* RDMA connection id */
 79	struct list_head     sc_accept_q;	/* Conn. waiting accept */
 80	struct rpcrdma_notification sc_rn;	/* removal notification */
 81	int		     sc_ord;		/* RDMA read limit */
 82	int                  sc_max_send_sges;
 83	bool		     sc_snd_w_inv;	/* OK to use Send With Invalidate */
 84
 85	atomic_t             sc_sq_avail;	/* SQEs ready to be consumed */
 86	unsigned int	     sc_sq_depth;	/* Depth of SQ */
 87	__be32		     sc_fc_credits;	/* Forward credits */
 88	u32		     sc_max_requests;	/* Max requests */
 89	u32		     sc_max_bc_requests;/* Backward credits */
 90	int                  sc_max_req_size;	/* Size of each RQ WR buf */
 91	u8		     sc_port_num;
 92
 93	struct ib_pd         *sc_pd;
 94
 95	spinlock_t	     sc_send_lock;
 96	struct llist_head    sc_send_ctxts;
 97	spinlock_t	     sc_rw_ctxt_lock;
 98	struct llist_head    sc_rw_ctxts;
 99
100	u32		     sc_pending_recvs;
101	u32		     sc_recv_batch;
102	struct list_head     sc_rq_dto_q;
103	struct list_head     sc_read_complete_q;
104	spinlock_t	     sc_rq_dto_lock;
105	struct ib_qp         *sc_qp;
106	struct ib_cq         *sc_rq_cq;
107	struct ib_cq         *sc_sq_cq;
108
109	spinlock_t	     sc_lock;		/* transport lock */
110
111	wait_queue_head_t    sc_send_wait;	/* SQ exhaustion waitlist */
112	unsigned long	     sc_flags;
113	struct work_struct   sc_work;
114
115	struct llist_head    sc_recv_ctxts;
116
117	atomic_t	     sc_completion_ids;
118};
119/* sc_flags */
120#define RDMAXPRT_CONN_PENDING	3
121
122static inline struct svcxprt_rdma *svc_rdma_rqst_rdma(struct svc_rqst *rqstp)
123{
124	struct svc_xprt *xprt = rqstp->rq_xprt;
125
126	return container_of(xprt, struct svcxprt_rdma, sc_xprt);
127}
128
129/*
130 * Default connection parameters
131 */
132enum {
133	RPCRDMA_LISTEN_BACKLOG	= 10,
134	RPCRDMA_MAX_REQUESTS	= 128,
135	RPCRDMA_MAX_BC_REQUESTS	= 2,
136};
137
138#define RPCSVC_MAXPAYLOAD_RDMA	RPCSVC_MAXPAYLOAD
139
140/**
141 * svc_rdma_send_cid_init - Initialize a Receive Queue completion ID
142 * @rdma: controlling transport
143 * @cid: completion ID to initialize
144 */
145static inline void svc_rdma_recv_cid_init(struct svcxprt_rdma *rdma,
146					  struct rpc_rdma_cid *cid)
147{
148	cid->ci_queue_id = rdma->sc_rq_cq->res.id;
149	cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids);
150}
151
152/**
153 * svc_rdma_send_cid_init - Initialize a Send Queue completion ID
154 * @rdma: controlling transport
155 * @cid: completion ID to initialize
156 */
157static inline void svc_rdma_send_cid_init(struct svcxprt_rdma *rdma,
158					  struct rpc_rdma_cid *cid)
159{
160	cid->ci_queue_id = rdma->sc_sq_cq->res.id;
161	cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids);
162}
163
164/*
165 * A chunk context tracks all I/O for moving one Read or Write
166 * chunk. This is a set of rdma_rw's that handle data movement
167 * for all segments of one chunk.
168 */
169struct svc_rdma_chunk_ctxt {
170	struct rpc_rdma_cid	cc_cid;
171	struct ib_cqe		cc_cqe;
172	struct list_head	cc_rwctxts;
173	ktime_t			cc_posttime;
174	int			cc_sqecount;
175};
176
177struct svc_rdma_recv_ctxt {
178	struct llist_node	rc_node;
179	struct list_head	rc_list;
180	struct ib_recv_wr	rc_recv_wr;
181	struct ib_cqe		rc_cqe;
182	struct rpc_rdma_cid	rc_cid;
183	struct ib_sge		rc_recv_sge;
184	void			*rc_recv_buf;
185	struct xdr_stream	rc_stream;
186	u32			rc_byte_len;
187	u32			rc_inv_rkey;
188	__be32			rc_msgtype;
189
190	/* State for pulling a Read chunk */
191	unsigned int		rc_pageoff;
192	unsigned int		rc_curpage;
193	unsigned int		rc_readbytes;
194	struct xdr_buf		rc_saved_arg;
195	struct svc_rdma_chunk_ctxt	rc_cc;
196
197	struct svc_rdma_pcl	rc_call_pcl;
198
199	struct svc_rdma_pcl	rc_read_pcl;
200	struct svc_rdma_chunk	*rc_cur_result_payload;
201	struct svc_rdma_pcl	rc_write_pcl;
202	struct svc_rdma_pcl	rc_reply_pcl;
203
204	unsigned int		rc_page_count;
205	unsigned long		rc_maxpages;
206	struct page		*rc_pages[] __counted_by(rc_maxpages);
207};
208
209/*
210 * State for sending a Write chunk.
211 *  - Tracks progress of writing one chunk over all its segments
212 *  - Stores arguments for the SGL constructor functions
213 */
214struct svc_rdma_write_info {
215	struct svcxprt_rdma	*wi_rdma;
216
217	const struct svc_rdma_chunk	*wi_chunk;
218
219	/* write state of this chunk */
220	unsigned int		wi_seg_off;
221	unsigned int		wi_seg_no;
222
223	/* SGL constructor arguments */
224	const struct xdr_buf	*wi_xdr;
225	unsigned char		*wi_base;
226	unsigned int		wi_next_off;
227
228	struct svc_rdma_chunk_ctxt	wi_cc;
229	struct work_struct	wi_work;
230};
231
232struct svc_rdma_send_ctxt {
233	struct llist_node	sc_node;
234	struct rpc_rdma_cid	sc_cid;
235	struct work_struct	sc_work;
236
237	struct svcxprt_rdma	*sc_rdma;
238	struct ib_send_wr	sc_send_wr;
239	struct ib_send_wr	*sc_wr_chain;
240	int			sc_sqecount;
241	struct ib_cqe		sc_cqe;
242	struct xdr_buf		sc_hdrbuf;
243	struct xdr_stream	sc_stream;
244	struct svc_rdma_write_info sc_reply_info;
245	void			*sc_xprt_buf;
246	int			sc_page_count;
247	int			sc_cur_sge_no;
248	unsigned long		sc_maxpages;
249	struct page		**sc_pages;
250	struct ib_sge		sc_sges[];
251};
252
253/* svc_rdma_backchannel.c */
254extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp,
255				     struct svc_rdma_recv_ctxt *rctxt);
256
257/* svc_rdma_recvfrom.c */
258extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma);
259extern bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma);
260extern struct svc_rdma_recv_ctxt *
261		svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma);
262extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
263				   struct svc_rdma_recv_ctxt *ctxt);
264extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma);
265extern void svc_rdma_release_ctxt(struct svc_xprt *xprt, void *ctxt);
266extern int svc_rdma_recvfrom(struct svc_rqst *);
267
268/* svc_rdma_rw.c */
269extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
270			     struct svc_rdma_chunk_ctxt *cc);
271extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
272extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
273			     struct svc_rdma_chunk_ctxt *cc);
274extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma,
275				struct svc_rdma_chunk_ctxt *cc,
276				enum dma_data_direction dir);
277extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma,
278					 struct svc_rdma_send_ctxt *ctxt);
279extern int svc_rdma_send_write_list(struct svcxprt_rdma *rdma,
280				    const struct svc_rdma_recv_ctxt *rctxt,
281				    const struct xdr_buf *xdr);
282extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma,
283					const struct svc_rdma_pcl *write_pcl,
284					const struct svc_rdma_pcl *reply_pcl,
285					struct svc_rdma_send_ctxt *sctxt,
286					const struct xdr_buf *xdr);
287extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
288				      struct svc_rqst *rqstp,
289				      struct svc_rdma_recv_ctxt *head);
290
291/* svc_rdma_sendto.c */
292extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma);
293extern struct svc_rdma_send_ctxt *
294		svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma);
295extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma,
296				   struct svc_rdma_send_ctxt *ctxt);
297extern int svc_rdma_post_send(struct svcxprt_rdma *rdma,
298			      struct svc_rdma_send_ctxt *ctxt);
299extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma,
300				  struct svc_rdma_send_ctxt *sctxt,
301				  const struct svc_rdma_pcl *write_pcl,
302				  const struct svc_rdma_pcl *reply_pcl,
303				  const struct xdr_buf *xdr);
304extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma,
305				    struct svc_rdma_send_ctxt *sctxt,
306				    struct svc_rdma_recv_ctxt *rctxt,
307				    int status);
308extern void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail);
309extern int svc_rdma_sendto(struct svc_rqst *);
310extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset,
311				   unsigned int length);
312
313/* svc_rdma_transport.c */
314extern struct svc_xprt_class svc_rdma_class;
315#ifdef CONFIG_SUNRPC_BACKCHANNEL
316extern struct svc_xprt_class svc_rdma_bc_class;
317#endif
318
319/* svc_rdma.c */
320extern int svc_rdma_init(void);
321extern void svc_rdma_cleanup(void);
322
323#endif