setup.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/* Copyright (c) 2019 Mellanox Technologies. */
  3
  4#include "setup.h"
  5#include "en/params.h"
  6#include "en/txrx.h"
  7#include "en/health.h"
  8#include <net/xdp_sock_drv.h>
  9
 10static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
 11					struct mlx5e_params *params,
 12					struct mlx5e_xsk_param *xsk)
 13{
 14	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
 15		mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
 16		return -EINVAL;
 17	}
 18
 19	return 0;
 20}
 21
 22/* The limitation of 2048 can be altered, but shouldn't go beyond the minimal
 23 * stride size of striding RQ.
 24 */
 25#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
 26
 27bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
 28			      struct mlx5e_xsk_param *xsk,
 29			      struct mlx5_core_dev *mdev)
 30{
 31	/* AF_XDP doesn't support frames larger than PAGE_SIZE,
 32	 * and xsk->chunk_size is limited to 65535 bytes.
 33	 */
 34	if ((size_t)xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) {
 35		mlx5_core_err(mdev, "XSK chunk size %u out of bounds [%u, %lu]\n", xsk->chunk_size,
 36			      MLX5E_MIN_XSK_CHUNK_SIZE, PAGE_SIZE);
 37		return false;
 38	}
 39
 40	/* frag_sz is different for regular and XSK RQs, so ensure that linear
 41	 * SKB mode is possible.
 42	 */
 43	switch (params->rq_wq_type) {
 44	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 45		return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
 46	default: /* MLX5_WQ_TYPE_CYCLIC */
 47		return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
 48	}
 49}
 50
 51static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
 52				   struct mlx5e_params *params,
 53				   struct mlx5e_xsk_param *xsk,
 54				   struct mlx5e_channel_param *cparam)
 55{
 56	mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
 57	mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
 58}
 59
 60static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
 61			     struct mlx5e_params *params,
 62			     struct xsk_buff_pool *pool,
 63			     struct mlx5e_xsk_param *xsk,
 64			     struct mlx5e_rq *rq)
 65{
 66	struct mlx5_core_dev *mdev = c->mdev;
 67	int rq_xdp_ix;
 68	int err;
 69
 70	rq->wq_type      = params->rq_wq_type;
 71	rq->pdev         = c->pdev;
 72	rq->netdev       = c->netdev;
 73	rq->priv         = c->priv;
 74	rq->hwtstamp_config = &c->priv->hwtstamp_config;
 75	rq->clock        = mdev->clock;
 76	rq->icosq        = &c->icosq;
 77	rq->ix           = c->ix;
 78	rq->channel      = c;
 79	rq->mdev         = mdev;
 80	rq->hw_mtu       = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 81	rq->xdpsq        = &c->rq_xdpsq;
 82	rq->xsk_pool     = pool;
 83	rq->stats        = &c->priv->channel_stats[c->ix]->xskrq;
 84	rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
 85	rq_xdp_ix        = c->ix;
 86	err = mlx5e_rq_set_handlers(rq, params, xsk);
 87	if (err)
 88		return err;
 89
 90	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
 91}
 92
 93static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
 94			     struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
 95			     struct mlx5e_xsk_param *xsk)
 96{
 97	u16 q_counter = c->priv->q_counter[c->sd_ix];
 98	struct mlx5e_rq *xskrq = &c->xskrq;
 99	int err;
100
101	err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
102	if (err)
103		return err;
104
105	err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
106	if (err)
107		return err;
108
109	__set_bit(MLX5E_RQ_STATE_XSK, &xskrq->state);
110	return 0;
111}
112
113int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
114		   struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
115		   struct mlx5e_channel *c)
116{
117	struct mlx5e_channel_param *cparam;
118	struct mlx5e_create_cq_param ccp;
119	int err;
120
121	mlx5e_build_create_cq_param(&ccp, c);
122
123	if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
124		return -EINVAL;
125
126	cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
127	if (!cparam)
128		return -ENOMEM;
129
130	mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
131
132	err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
133			    &c->xskrq.cq);
134	if (unlikely(err))
135		goto err_free_cparam;
136
137	err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
138	if (unlikely(err))
139		goto err_close_rx_cq;
140
141	err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
142			    &c->xsksq.cq);
143	if (unlikely(err))
144		goto err_close_rq;
145
146	/* Create a separate SQ, so that when the buff pool is disabled, we could
147	 * close this SQ safely and stop receiving CQEs. In other case, e.g., if
148	 * the XDPSQ was used instead, we might run into trouble when the buff pool
149	 * is disabled and then re-enabled, but the SQ continues receiving CQEs
150	 * from the old buff pool.
151	 */
152	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, pool, &c->xsksq, true);
153	if (unlikely(err))
154		goto err_close_tx_cq;
155
156	kvfree(cparam);
157
158	set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
159
160	return 0;
161
162err_close_tx_cq:
163	mlx5e_close_cq(&c->xsksq.cq);
164
165err_close_rq:
166	mlx5e_close_rq(&c->xskrq);
167
168err_close_rx_cq:
169	mlx5e_close_cq(&c->xskrq.cq);
170
171err_free_cparam:
172	kvfree(cparam);
173
174	return err;
175}
176
177void mlx5e_close_xsk(struct mlx5e_channel *c)
178{
179	clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
180	synchronize_net(); /* Sync with NAPI. */
181
182	mlx5e_close_rq(&c->xskrq);
183	mlx5e_close_cq(&c->xskrq.cq);
184	mlx5e_close_xdpsq(&c->xsksq);
185	mlx5e_close_cq(&c->xsksq.cq);
186
187	memset(&c->xskrq, 0, sizeof(c->xskrq));
188	memset(&c->xsksq, 0, sizeof(c->xsksq));
189}
190
191void mlx5e_activate_xsk(struct mlx5e_channel *c)
192{
193	/* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid
194	 * activating XSKRQ in the middle of recovery.
195	 */
196	mlx5e_reporter_icosq_suspend_recovery(c);
197	set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
198	mlx5e_reporter_icosq_resume_recovery(c);
199
200	/* TX queue is created active. */
201}
202
203void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
204{
205	/* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the
206	 * middle of recovery. Suspend the recovery to avoid it.
207	 */
208	mlx5e_reporter_icosq_suspend_recovery(c);
209	clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
210	mlx5e_reporter_icosq_resume_recovery(c);
211	synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
212
213	/* TX queue is disabled on close. */
214}