netdev_rx_queue.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#include <linux/ethtool_netlink.h>
  4#include <linux/netdevice.h>
  5#include <net/netdev_lock.h>
  6#include <net/netdev_queues.h>
  7#include <net/netdev_rx_queue.h>
  8#include <net/page_pool/memory_provider.h>
  9
 10#include "page_pool_priv.h"
 11
 12/* See also page_pool_is_unreadable() */
 13bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx)
 14{
 15	struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, idx);
 16
 17	return !!rxq->mp_params.mp_ops;
 18}
 19EXPORT_SYMBOL(netif_rxq_has_unreadable_mp);
 20
 21int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx)
 22{
 23	struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx);
 24	const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops;
 25	void *new_mem, *old_mem;
 26	int err;
 27
 28	if (!qops || !qops->ndo_queue_stop || !qops->ndo_queue_mem_free ||
 29	    !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start)
 30		return -EOPNOTSUPP;
 31
 32	netdev_assert_locked(dev);
 33
 34	new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
 35	if (!new_mem)
 36		return -ENOMEM;
 37
 38	old_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL);
 39	if (!old_mem) {
 40		err = -ENOMEM;
 41		goto err_free_new_mem;
 42	}
 43
 44	err = qops->ndo_queue_mem_alloc(dev, new_mem, rxq_idx);
 45	if (err)
 46		goto err_free_old_mem;
 47
 48	err = page_pool_check_memory_provider(dev, rxq);
 49	if (err)
 50		goto err_free_new_queue_mem;
 51
 52	if (netif_running(dev)) {
 53		err = qops->ndo_queue_stop(dev, old_mem, rxq_idx);
 54		if (err)
 55			goto err_free_new_queue_mem;
 56
 57		err = qops->ndo_queue_start(dev, new_mem, rxq_idx);
 58		if (err)
 59			goto err_start_queue;
 60	} else {
 61		swap(new_mem, old_mem);
 62	}
 63
 64	qops->ndo_queue_mem_free(dev, old_mem);
 65
 66	kvfree(old_mem);
 67	kvfree(new_mem);
 68
 69	return 0;
 70
 71err_start_queue:
 72	/* Restarting the queue with old_mem should be successful as we haven't
 73	 * changed any of the queue configuration, and there is not much we can
 74	 * do to recover from a failure here.
 75	 *
 76	 * WARN if we fail to recover the old rx queue, and at least free
 77	 * old_mem so we don't also leak that.
 78	 */
 79	if (qops->ndo_queue_start(dev, old_mem, rxq_idx)) {
 80		WARN(1,
 81		     "Failed to restart old queue in error path. RX queue %d may be unhealthy.",
 82		     rxq_idx);
 83		qops->ndo_queue_mem_free(dev, old_mem);
 84	}
 85
 86err_free_new_queue_mem:
 87	qops->ndo_queue_mem_free(dev, new_mem);
 88
 89err_free_old_mem:
 90	kvfree(old_mem);
 91
 92err_free_new_mem:
 93	kvfree(new_mem);
 94
 95	return err;
 96}
 97EXPORT_SYMBOL_NS_GPL(netdev_rx_queue_restart, "NETDEV_INTERNAL");
 98
 99int __net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
100		      const struct pp_memory_provider_params *p,
101		      struct netlink_ext_ack *extack)
102{
103	struct netdev_rx_queue *rxq;
104	int ret;
105
106	if (!netdev_need_ops_lock(dev))
107		return -EOPNOTSUPP;
108
109	if (rxq_idx >= dev->real_num_rx_queues) {
110		NL_SET_ERR_MSG(extack, "rx queue index out of range");
111		return -ERANGE;
112	}
113	rxq_idx = array_index_nospec(rxq_idx, dev->real_num_rx_queues);
114
115	if (dev->cfg->hds_config != ETHTOOL_TCP_DATA_SPLIT_ENABLED) {
116		NL_SET_ERR_MSG(extack, "tcp-data-split is disabled");
117		return -EINVAL;
118	}
119	if (dev->cfg->hds_thresh) {
120		NL_SET_ERR_MSG(extack, "hds-thresh is not zero");
121		return -EINVAL;
122	}
123	if (dev_xdp_prog_count(dev)) {
124		NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached");
125		return -EEXIST;
126	}
127
128	rxq = __netif_get_rx_queue(dev, rxq_idx);
129	if (rxq->mp_params.mp_ops) {
130		NL_SET_ERR_MSG(extack, "designated queue already memory provider bound");
131		return -EEXIST;
132	}
133#ifdef CONFIG_XDP_SOCKETS
134	if (rxq->pool) {
135		NL_SET_ERR_MSG(extack, "designated queue already in use by AF_XDP");
136		return -EBUSY;
137	}
138#endif
139
140	rxq->mp_params = *p;
141	ret = netdev_rx_queue_restart(dev, rxq_idx);
142	if (ret) {
143		rxq->mp_params.mp_ops = NULL;
144		rxq->mp_params.mp_priv = NULL;
145	}
146	return ret;
147}
148
149int net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx,
150		    struct pp_memory_provider_params *p)
151{
152	int ret;
153
154	netdev_lock(dev);
155	ret = __net_mp_open_rxq(dev, rxq_idx, p, NULL);
156	netdev_unlock(dev);
157	return ret;
158}
159
160void __net_mp_close_rxq(struct net_device *dev, unsigned int ifq_idx,
161			const struct pp_memory_provider_params *old_p)
162{
163	struct netdev_rx_queue *rxq;
164	int err;
165
166	if (WARN_ON_ONCE(ifq_idx >= dev->real_num_rx_queues))
167		return;
168
169	rxq = __netif_get_rx_queue(dev, ifq_idx);
170
171	/* Callers holding a netdev ref may get here after we already
172	 * went thru shutdown via dev_memory_provider_uninstall().
173	 */
174	if (dev->reg_state > NETREG_REGISTERED &&
175	    !rxq->mp_params.mp_ops)
176		return;
177
178	if (WARN_ON_ONCE(rxq->mp_params.mp_ops != old_p->mp_ops ||
179			 rxq->mp_params.mp_priv != old_p->mp_priv))
180		return;
181
182	rxq->mp_params.mp_ops = NULL;
183	rxq->mp_params.mp_priv = NULL;
184	err = netdev_rx_queue_restart(dev, ifq_idx);
185	WARN_ON(err && err != -ENETDOWN);
186}
187
188void net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx,
189		      struct pp_memory_provider_params *old_p)
190{
191	netdev_lock(dev);
192	__net_mp_close_rxq(dev, ifq_idx, old_p);
193	netdev_unlock(dev);
194}