xdp.h

  1/*
  2 * Copyright (c) 2018, Mellanox Technologies. All rights reserved.
  3 *
  4 * This software is available to you under a choice of one of two
  5 * licenses.  You may choose to be licensed under the terms of the GNU
  6 * General Public License (GPL) Version 2, available from the file
  7 * COPYING in the main directory of this source tree, or the
  8 * OpenIB.org BSD license below:
  9 *
 10 *     Redistribution and use in source and binary forms, with or
 11 *     without modification, are permitted provided that the following
 12 *     conditions are met:
 13 *
 14 *      - Redistributions of source code must retain the above
 15 *        copyright notice, this list of conditions and the following
 16 *        disclaimer.
 17 *
 18 *      - Redistributions in binary form must reproduce the above
 19 *        copyright notice, this list of conditions and the following
 20 *        disclaimer in the documentation and/or other materials
 21 *        provided with the distribution.
 22 *
 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 30 * SOFTWARE.
 31 */
 32#ifndef __MLX5_EN_XDP_H__
 33#define __MLX5_EN_XDP_H__
 34
 35#include <linux/indirect_call_wrapper.h>
 36#include <net/xdp_sock.h>
 37
 38#include "en.h"
 39#include "en/txrx.h"
 40
 41#define MLX5E_XDP_MIN_INLINE (ETH_HLEN + VLAN_HLEN)
 42
 43#define MLX5E_XDP_INLINE_WQE_MAX_DS_CNT 16
 44#define MLX5E_XDP_INLINE_WQE_SZ_THRSD \
 45	(MLX5E_XDP_INLINE_WQE_MAX_DS_CNT * MLX5_SEND_WQE_DS - \
 46	 sizeof(struct mlx5_wqe_inline_seg))
 47
 48/* XDP packets can be transmitted in different ways. On completion, we need to
 49 * distinguish between them to clean up things in a proper way.
 50 */
 51enum mlx5e_xdp_xmit_mode {
 52	/* An xdp_frame was transmitted due to either XDP_REDIRECT from another
 53	 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
 54	 * returned.
 55	 */
 56	MLX5E_XDP_XMIT_MODE_FRAME,
 57
 58	/* The xdp_frame was created in place as a result of XDP_TX from a
 59	 * regular RQ. No DMA remapping happened, and the page belongs to us.
 60	 */
 61	MLX5E_XDP_XMIT_MODE_PAGE,
 62
 63	/* No xdp_frame was created at all, the transmit happened from a UMEM
 64	 * page. The UMEM Completion Ring producer pointer has to be increased.
 65	 */
 66	MLX5E_XDP_XMIT_MODE_XSK,
 67};
 68
 69/* xmit_mode entry is pushed to the fifo per packet, followed by multiple
 70 * entries, as follows:
 71 *
 72 * MLX5E_XDP_XMIT_MODE_FRAME:
 73 *    xdpf, dma_addr_1, dma_addr_2, ... , dma_addr_num.
 74 *    'num' is derived from xdpf.
 75 *
 76 * MLX5E_XDP_XMIT_MODE_PAGE:
 77 *    num, page_1, page_2, ... , page_num.
 78 *
 79 * MLX5E_XDP_XMIT_MODE_XSK:
 80 *    frame.xsk_meta.
 81 */
 82#define MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO 4
 83
 84union mlx5e_xdp_info {
 85	enum mlx5e_xdp_xmit_mode mode;
 86	union {
 87		struct xdp_frame *xdpf;
 88		dma_addr_t dma_addr;
 89	} frame;
 90	union {
 91		struct mlx5e_rq *rq;
 92		u8 num;
 93		struct page *page;
 94	} page;
 95	struct xsk_tx_metadata_compl xsk_meta;
 96};
 97
 98struct mlx5e_xsk_param;
 99int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
100bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
101		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
102void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
103bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
104void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
105void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw);
106void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq);
107int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
108		   u32 flags);
109
110extern const struct xdp_metadata_ops mlx5e_xdp_metadata_ops;
111extern const struct xsk_tx_metadata_ops mlx5e_xsk_tx_metadata_ops;
112
113INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq,
114							  struct mlx5e_xmit_data *xdptxd,
115							  int check_result,
116							  struct xsk_tx_metadata *meta));
117INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
118						    struct mlx5e_xmit_data *xdptxd,
119						    int check_result,
120						    struct xsk_tx_metadata *meta));
121INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq));
122INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq));
123
124static inline void mlx5e_xdp_tx_enable(struct mlx5e_priv *priv)
125{
126	set_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
127
128	if (priv->channels.params.xdp_prog)
129		set_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
130}
131
132static inline void mlx5e_xdp_tx_disable(struct mlx5e_priv *priv)
133{
134	if (priv->channels.params.xdp_prog)
135		clear_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
136
137	clear_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
138	/* Let other device's napi(s) and XSK wakeups see our new state. */
139	synchronize_net();
140}
141
142static inline bool mlx5e_xdp_tx_is_enabled(struct mlx5e_priv *priv)
143{
144	return test_bit(MLX5E_STATE_XDP_TX_ENABLED, &priv->state);
145}
146
147static inline bool mlx5e_xdp_is_active(struct mlx5e_priv *priv)
148{
149	return test_bit(MLX5E_STATE_XDP_ACTIVE, &priv->state);
150}
151
152static inline void mlx5e_xmit_xdp_doorbell(struct mlx5e_xdpsq *sq)
153{
154	if (sq->doorbell_cseg) {
155		mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, sq->doorbell_cseg);
156		sq->doorbell_cseg = NULL;
157	}
158}
159
160/* Enable inline WQEs to shift some load from a congested HCA (HW) to
161 * a less congested cpu (SW).
162 */
163static inline bool mlx5e_xdp_get_inline_state(struct mlx5e_xdpsq *sq, bool cur)
164{
165	u16 outstanding = sq->xdpi_fifo_pc - sq->xdpi_fifo_cc;
166
167#define MLX5E_XDP_INLINE_WATERMARK_LOW	10
168#define MLX5E_XDP_INLINE_WATERMARK_HIGH 128
169
170	if (cur && outstanding <= MLX5E_XDP_INLINE_WATERMARK_LOW)
171		return false;
172
173	if (!cur && outstanding >= MLX5E_XDP_INLINE_WATERMARK_HIGH)
174		return true;
175
176	return cur;
177}
178
179static inline bool mlx5e_xdp_mpwqe_is_full(struct mlx5e_tx_mpwqe *session)
180{
181	if (session->inline_on)
182		return session->ds_count + MLX5E_XDP_INLINE_WQE_MAX_DS_CNT >
183		       session->ds_count_max;
184
185	return mlx5e_tx_mpwqe_is_full(session);
186}
187
188struct mlx5e_xdp_wqe_info {
189	u8 num_wqebbs;
190	u8 num_pkts;
191};
192
193static inline void
194mlx5e_xdp_mpwqe_add_dseg(struct mlx5e_xdpsq *sq,
195			 struct mlx5e_xmit_data *xdptxd,
196			 struct mlx5e_xdpsq_stats *stats)
197{
198	struct mlx5e_tx_mpwqe *session = &sq->mpwqe;
199	struct mlx5_wqe_data_seg *dseg =
200		(struct mlx5_wqe_data_seg *)session->wqe + session->ds_count;
201	u32 dma_len = xdptxd->len;
202
203	session->pkt_count++;
204	session->bytes_count += dma_len;
205
206	if (session->inline_on && dma_len <= MLX5E_XDP_INLINE_WQE_SZ_THRSD) {
207		struct mlx5_wqe_inline_seg *inline_dseg =
208			(struct mlx5_wqe_inline_seg *)dseg;
209		u16 ds_len = sizeof(*inline_dseg) + dma_len;
210		u16 ds_cnt = DIV_ROUND_UP(ds_len, MLX5_SEND_WQE_DS);
211
212		inline_dseg->byte_count = cpu_to_be32(dma_len | MLX5_INLINE_SEG);
213		memcpy(inline_dseg->data, xdptxd->data, dma_len);
214
215		session->ds_count += ds_cnt;
216		stats->inlnw++;
217		return;
218	}
219
220	dseg->addr       = cpu_to_be64(xdptxd->dma_addr);
221	dseg->byte_count = cpu_to_be32(dma_len);
222	dseg->lkey       = sq->mkey_be;
223	session->ds_count++;
224}
225
226static inline void
227mlx5e_xdpi_fifo_push(struct mlx5e_xdp_info_fifo *fifo,
228		     union mlx5e_xdp_info xi)
229{
230	u32 i = (*fifo->pc)++ & fifo->mask;
231
232	fifo->xi[i] = xi;
233}
234
235static inline union mlx5e_xdp_info
236mlx5e_xdpi_fifo_pop(struct mlx5e_xdp_info_fifo *fifo)
237{
238	return fifo->xi[(*fifo->cc)++ & fifo->mask];
239}
240#endif