rxe_queue.h

  1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2/*
  3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
  4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
  5 */
  6
  7#ifndef RXE_QUEUE_H
  8#define RXE_QUEUE_H
  9
 10/* Implements a simple circular buffer that is shared between user
 11 * and the driver and can be resized. The requested element size is
 12 * rounded up to a power of 2 and the number of elements in the buffer
 13 * is also rounded up to a power of 2. Since the queue is empty when
 14 * the producer and consumer indices match the maximum capacity of the
 15 * queue is one less than the number of element slots.
 16 *
 17 * Notes:
 18 *   - The driver indices are always masked off to q->index_mask
 19 *     before storing so do not need to be checked on reads.
 20 *   - The user whether user space or kernel is generally
 21 *     not trusted so its parameters are masked to make sure
 22 *     they do not access the queue out of bounds on reads.
 23 *   - The driver indices for queues must not be written
 24 *     by user so a local copy is used and a shared copy is
 25 *     stored when the local copy is changed.
 26 *   - By passing the type in the parameter list separate from q
 27 *     the compiler can eliminate the switch statement when the
 28 *     actual queue type is known when the function is called at
 29 *     compile time.
 30 *   - These queues are lock free. The user and driver must protect
 31 *     changes to their end of the queues with locks if more than one
 32 *     CPU can be accessing it at the same time.
 33 */
 34
 35/**
 36 * enum queue_type - type of queue
 37 * @QUEUE_TYPE_TO_CLIENT:	Queue is written by rxe driver and
 38 *				read by client which may be a user space
 39 *				application or a kernel ulp.
 40 *				Used by rxe internals only.
 41 * @QUEUE_TYPE_FROM_CLIENT:	Queue is written by client and
 42 *				read by rxe driver.
 43 *				Used by rxe internals only.
 44 * @QUEUE_TYPE_FROM_ULP:	Queue is written by kernel ulp and
 45 *				read by rxe driver.
 46 *				Used by kernel verbs APIs only on
 47 *				behalf of ulps.
 48 * @QUEUE_TYPE_TO_ULP:		Queue is written by rxe driver and
 49 *				read by kernel ulp.
 50 *				Used by kernel verbs APIs only on
 51 *				behalf of ulps.
 52 */
 53enum queue_type {
 54	QUEUE_TYPE_TO_CLIENT,
 55	QUEUE_TYPE_FROM_CLIENT,
 56	QUEUE_TYPE_FROM_ULP,
 57	QUEUE_TYPE_TO_ULP,
 58};
 59
 60struct rxe_queue_buf;
 61
 62struct rxe_queue {
 63	struct rxe_dev		*rxe;
 64	struct rxe_queue_buf	*buf;
 65	struct rxe_mmap_info	*ip;
 66	size_t			buf_size;
 67	size_t			elem_size;
 68	unsigned int		log2_elem_size;
 69	u32			index_mask;
 70	enum queue_type		type;
 71	/* private copy of index for shared queues between
 72	 * driver and clients. Driver reads and writes
 73	 * this copy and then replicates to rxe_queue_buf
 74	 * for read access by clients.
 75	 */
 76	u32			index;
 77};
 78
 79int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
 80		 struct ib_udata *udata, struct rxe_queue_buf *buf,
 81		 size_t buf_size, struct rxe_mmap_info **ip_p);
 82
 83void rxe_queue_reset(struct rxe_queue *q);
 84
 85struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
 86			unsigned int elem_size, enum queue_type type);
 87
 88int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
 89		     unsigned int elem_size, struct ib_udata *udata,
 90		     struct mminfo __user *outbuf,
 91		     spinlock_t *producer_lock, spinlock_t *consumer_lock);
 92
 93void rxe_queue_cleanup(struct rxe_queue *queue);
 94
 95static inline u32 queue_next_index(struct rxe_queue *q, int index)
 96{
 97	return (index + 1) & q->index_mask;
 98}
 99
100static inline u32 queue_get_producer(const struct rxe_queue *q,
101				     enum queue_type type)
102{
103	u32 prod;
104
105	switch (type) {
106	case QUEUE_TYPE_FROM_CLIENT:
107		/* used by rxe, client owns the index */
108		prod = smp_load_acquire(&q->buf->producer_index);
109		break;
110	case QUEUE_TYPE_TO_CLIENT:
111		/* used by rxe which owns the index */
112		prod = q->index;
113		break;
114	case QUEUE_TYPE_FROM_ULP:
115		/* used by ulp which owns the index */
116		prod = q->buf->producer_index;
117		break;
118	case QUEUE_TYPE_TO_ULP:
119		/* used by ulp, rxe owns the index */
120		prod = smp_load_acquire(&q->buf->producer_index);
121		break;
122	}
123
124	return prod;
125}
126
127static inline u32 queue_get_consumer(const struct rxe_queue *q,
128				     enum queue_type type)
129{
130	u32 cons;
131
132	switch (type) {
133	case QUEUE_TYPE_FROM_CLIENT:
134		/* used by rxe which owns the index */
135		cons = q->index;
136		break;
137	case QUEUE_TYPE_TO_CLIENT:
138		/* used by rxe, client owns the index */
139		cons = smp_load_acquire(&q->buf->consumer_index);
140		break;
141	case QUEUE_TYPE_FROM_ULP:
142		/* used by ulp, rxe owns the index */
143		cons = smp_load_acquire(&q->buf->consumer_index);
144		break;
145	case QUEUE_TYPE_TO_ULP:
146		/* used by ulp which owns the index */
147		cons = q->buf->consumer_index;
148		break;
149	}
150
151	return cons;
152}
153
154static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
155{
156	u32 prod = queue_get_producer(q, type);
157	u32 cons = queue_get_consumer(q, type);
158
159	return ((prod - cons) & q->index_mask) == 0;
160}
161
162static inline int queue_full(struct rxe_queue *q, enum queue_type type)
163{
164	u32 prod = queue_get_producer(q, type);
165	u32 cons = queue_get_consumer(q, type);
166
167	return ((prod + 1 - cons) & q->index_mask) == 0;
168}
169
170static inline u32 queue_count(const struct rxe_queue *q,
171					enum queue_type type)
172{
173	u32 prod = queue_get_producer(q, type);
174	u32 cons = queue_get_consumer(q, type);
175
176	return (prod - cons) & q->index_mask;
177}
178
179static inline void queue_advance_producer(struct rxe_queue *q,
180					  enum queue_type type)
181{
182	u32 prod;
183
184	switch (type) {
185	case QUEUE_TYPE_FROM_CLIENT:
186		/* used by rxe, client owns the index */
187		if (WARN_ON(1))
188			pr_warn("%s: attempt to advance client index\n",
189				__func__);
190		break;
191	case QUEUE_TYPE_TO_CLIENT:
192		/* used by rxe which owns the index */
193		prod = q->index;
194		prod = (prod + 1) & q->index_mask;
195		q->index = prod;
196		/* release so client can read it safely */
197		smp_store_release(&q->buf->producer_index, prod);
198		break;
199	case QUEUE_TYPE_FROM_ULP:
200		/* used by ulp which owns the index */
201		prod = q->buf->producer_index;
202		prod = (prod + 1) & q->index_mask;
203		/* release so rxe can read it safely */
204		smp_store_release(&q->buf->producer_index, prod);
205		break;
206	case QUEUE_TYPE_TO_ULP:
207		/* used by ulp, rxe owns the index */
208		if (WARN_ON(1))
209			pr_warn("%s: attempt to advance driver index\n",
210				__func__);
211		break;
212	}
213}
214
215static inline void queue_advance_consumer(struct rxe_queue *q,
216					  enum queue_type type)
217{
218	u32 cons;
219
220	switch (type) {
221	case QUEUE_TYPE_FROM_CLIENT:
222		/* used by rxe which owns the index */
223		cons = (q->index + 1) & q->index_mask;
224		q->index = cons;
225		/* release so client can read it safely */
226		smp_store_release(&q->buf->consumer_index, cons);
227		break;
228	case QUEUE_TYPE_TO_CLIENT:
229		/* used by rxe, client owns the index */
230		if (WARN_ON(1))
231			pr_warn("%s: attempt to advance client index\n",
232				__func__);
233		break;
234	case QUEUE_TYPE_FROM_ULP:
235		/* used by ulp, rxe owns the index */
236		if (WARN_ON(1))
237			pr_warn("%s: attempt to advance driver index\n",
238				__func__);
239		break;
240	case QUEUE_TYPE_TO_ULP:
241		/* used by ulp which owns the index */
242		cons = q->buf->consumer_index;
243		cons = (cons + 1) & q->index_mask;
244		/* release so rxe can read it safely */
245		smp_store_release(&q->buf->consumer_index, cons);
246		break;
247	}
248}
249
250static inline void *queue_producer_addr(struct rxe_queue *q,
251					enum queue_type type)
252{
253	u32 prod = queue_get_producer(q, type);
254
255	return q->buf->data + (prod << q->log2_elem_size);
256}
257
258static inline void *queue_consumer_addr(struct rxe_queue *q,
259					enum queue_type type)
260{
261	u32 cons = queue_get_consumer(q, type);
262
263	return q->buf->data + (cons << q->log2_elem_size);
264}
265
266static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
267{
268	return q->buf->data + ((index & q->index_mask)
269				<< q->log2_elem_size);
270}
271
272static inline u32 queue_index_from_addr(const struct rxe_queue *q,
273				const void *addr)
274{
275	return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
276				& q->index_mask;
277}
278
279static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
280{
281	return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
282}
283
284#endif /* RXE_QUEUE_H */