tx.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2019 Mellanox Technologies. */
  3
  4#include "tx.h"
  5#include "pool.h"
  6#include "en/xdp.h"
  7#include "en/params.h"
  8#include <net/xdp_sock_drv.h>
  9
 10int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
 11{
 12	struct mlx5e_priv *priv = netdev_priv(dev);
 13	struct mlx5e_params *params = &priv->channels.params;
 14	struct mlx5e_channel *c;
 15
 16	if (unlikely(!mlx5e_xdp_is_active(priv)))
 17		return -ENETDOWN;
 18
 19	if (unlikely(qid >= params->num_channels))
 20		return -EINVAL;
 21
 22	c = priv->channels.c[qid];
 23
 24	if (!napi_if_scheduled_mark_missed(&c->napi)) {
 25		/* To avoid WQE overrun, don't post a NOP if async_icosq is not
 26		 * active and not polled by NAPI. Return 0, because the upcoming
 27		 * activate will trigger the IRQ for us.
 28		 */
 29		if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state)))
 30			return 0;
 31
 32		if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state))
 33			return 0;
 34
 35		mlx5e_trigger_napi_icosq(c);
 36	}
 37
 38	return 0;
 39}
 40
 41/* When TX fails (because of the size of the packet), we need to get completions
 42 * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish
 43 * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the
 44 * same.
 45 */
 46static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq,
 47				  union mlx5e_xdp_info *xdpi)
 48{
 49	u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
 50	struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi];
 51	struct mlx5e_tx_wqe *nopwqe;
 52
 53	wi->num_wqebbs = 1;
 54	wi->num_pkts = 1;
 55
 56	nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc);
 57	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, *xdpi);
 58	if (xp_tx_metadata_enabled(sq->xsk_pool))
 59		mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
 60				     (union mlx5e_xdp_info) { .xsk_meta = {} });
 61	sq->doorbell_cseg = &nopwqe->ctrl;
 62}
 63
 64bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget)
 65{
 66	struct xsk_buff_pool *pool = sq->xsk_pool;
 67	struct xsk_tx_metadata *meta = NULL;
 68	union mlx5e_xdp_info xdpi;
 69	bool work_done = true;
 70	bool flush = false;
 71
 72	xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK;
 73
 74	for (; budget; budget--) {
 75		int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check,
 76						   mlx5e_xmit_xdp_frame_check_mpwqe,
 77						   mlx5e_xmit_xdp_frame_check,
 78						   sq);
 79		struct mlx5e_xmit_data xdptxd = {};
 80		struct xdp_desc desc;
 81		bool ret;
 82
 83		if (unlikely(check_result < 0)) {
 84			work_done = false;
 85			break;
 86		}
 87
 88		if (!xsk_tx_peek_desc(pool, &desc)) {
 89			/* TX will get stuck until something wakes it up by
 90			 * triggering NAPI. Currently it's expected that the
 91			 * application calls sendto() if there are consumed, but
 92			 * not completed frames.
 93			 */
 94			break;
 95		}
 96
 97		xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr);
 98		xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr);
 99		xdptxd.len = desc.len;
100		meta = xsk_buff_get_metadata(pool, desc.addr);
101
102		xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len);
103
104		ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe,
105				      mlx5e_xmit_xdp_frame, sq, &xdptxd,
106				      check_result, meta);
107		if (unlikely(!ret)) {
108			if (sq->mpwqe.wqe)
109				mlx5e_xdp_mpwqe_complete(sq);
110
111			mlx5e_xsk_tx_post_err(sq, &xdpi);
112		} else {
113			mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi);
114			if (xp_tx_metadata_enabled(sq->xsk_pool)) {
115				struct xsk_tx_metadata_compl compl;
116
117				xsk_tx_metadata_to_compl(meta, &compl);
118				XSK_TX_COMPL_FITS(void *);
119
120				mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
121						     (union mlx5e_xdp_info)
122						     { .xsk_meta = compl });
123			}
124		}
125
126		flush = true;
127	}
128
129	if (flush) {
130		if (sq->mpwqe.wqe)
131			mlx5e_xdp_mpwqe_complete(sq);
132		mlx5e_xmit_xdp_doorbell(sq);
133
134		xsk_tx_release(pool);
135	}
136
137	return !(budget && work_done);
138}