devmem.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 *      Devmem TCP
  4 *
  5 *      Authors:	Mina Almasry <almasrymina@google.com>
  6 *			Willem de Bruijn <willemdebruijn.kernel@gmail.com>
  7 *			Kaiyuan Zhang <kaiyuanz@google.com
  8 */
  9
 10#include <linux/dma-buf.h>
 11#include <linux/genalloc.h>
 12#include <linux/mm.h>
 13#include <linux/netdevice.h>
 14#include <linux/types.h>
 15#include <net/netdev_queues.h>
 16#include <net/netdev_rx_queue.h>
 17#include <net/page_pool/helpers.h>
 18#include <net/page_pool/memory_provider.h>
 19#include <net/sock.h>
 20#include <net/tcp.h>
 21#include <trace/events/page_pool.h>
 22
 23#include "devmem.h"
 24#include "mp_dmabuf_devmem.h"
 25#include "page_pool_priv.h"
 26
 27/* Device memory support */
 28
 29static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
 30
 31static const struct memory_provider_ops dmabuf_devmem_ops;
 32
 33bool net_is_devmem_iov(struct net_iov *niov)
 34{
 35	return niov->type == NET_IOV_DMABUF;
 36}
 37
 38static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
 39					       struct gen_pool_chunk *chunk,
 40					       void *not_used)
 41{
 42	struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
 43
 44	kvfree(owner->area.niovs);
 45	kfree(owner);
 46}
 47
 48static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
 49{
 50	struct dmabuf_genpool_chunk_owner *owner;
 51
 52	owner = net_devmem_iov_to_chunk_owner(niov);
 53	return owner->base_dma_addr +
 54	       ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
 55}
 56
 57void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
 58{
 59	struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
 60
 61	size_t size, avail;
 62
 63	gen_pool_for_each_chunk(binding->chunk_pool,
 64				net_devmem_dmabuf_free_chunk_owner, NULL);
 65
 66	size = gen_pool_size(binding->chunk_pool);
 67	avail = gen_pool_avail(binding->chunk_pool);
 68
 69	if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
 70		  size, avail))
 71		gen_pool_destroy(binding->chunk_pool);
 72
 73	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
 74					  binding->direction);
 75	dma_buf_detach(binding->dmabuf, binding->attachment);
 76	dma_buf_put(binding->dmabuf);
 77	xa_destroy(&binding->bound_rxqs);
 78	kvfree(binding->tx_vec);
 79	kfree(binding);
 80}
 81
 82struct net_iov *
 83net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
 84{
 85	struct dmabuf_genpool_chunk_owner *owner;
 86	unsigned long dma_addr;
 87	struct net_iov *niov;
 88	ssize_t offset;
 89	ssize_t index;
 90
 91	dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
 92					(void **)&owner);
 93	if (!dma_addr)
 94		return NULL;
 95
 96	offset = dma_addr - owner->base_dma_addr;
 97	index = offset / PAGE_SIZE;
 98	niov = &owner->area.niovs[index];
 99
100	niov->desc.pp_magic = 0;
101	niov->desc.pp = NULL;
102	atomic_long_set(&niov->desc.pp_ref_count, 0);
103
104	return niov;
105}
106
107void net_devmem_free_dmabuf(struct net_iov *niov)
108{
109	struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
110	unsigned long dma_addr = net_devmem_get_dma_addr(niov);
111
112	if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
113				       PAGE_SIZE)))
114		return;
115
116	gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
117}
118
119void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
120{
121	struct netdev_rx_queue *rxq;
122	unsigned long xa_idx;
123	unsigned int rxq_idx;
124
125	xa_erase(&net_devmem_dmabuf_bindings, binding->id);
126
127	/* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
128	 * erase.
129	 */
130	synchronize_net();
131
132	if (binding->list.next)
133		list_del(&binding->list);
134
135	xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
136		const struct pp_memory_provider_params mp_params = {
137			.mp_priv	= binding,
138			.mp_ops		= &dmabuf_devmem_ops,
139		};
140
141		rxq_idx = get_netdev_rx_queue_index(rxq);
142
143		__net_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
144	}
145
146	net_devmem_dmabuf_binding_put(binding);
147}
148
149int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
150				    struct net_devmem_dmabuf_binding *binding,
151				    struct netlink_ext_ack *extack)
152{
153	struct pp_memory_provider_params mp_params = {
154		.mp_priv	= binding,
155		.mp_ops		= &dmabuf_devmem_ops,
156	};
157	struct netdev_rx_queue *rxq;
158	u32 xa_idx;
159	int err;
160
161	err = __net_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
162	if (err)
163		return err;
164
165	rxq = __netif_get_rx_queue(dev, rxq_idx);
166	err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
167		       GFP_KERNEL);
168	if (err)
169		goto err_close_rxq;
170
171	return 0;
172
173err_close_rxq:
174	__net_mp_close_rxq(dev, rxq_idx, &mp_params);
175	return err;
176}
177
178struct net_devmem_dmabuf_binding *
179net_devmem_bind_dmabuf(struct net_device *dev,
180		       struct device *dma_dev,
181		       enum dma_data_direction direction,
182		       unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
183		       struct netlink_ext_ack *extack)
184{
185	struct net_devmem_dmabuf_binding *binding;
186	static u32 id_alloc_next;
187	struct scatterlist *sg;
188	struct dma_buf *dmabuf;
189	unsigned int sg_idx, i;
190	unsigned long virtual;
191	int err;
192
193	if (!dma_dev) {
194		NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
195		return ERR_PTR(-EOPNOTSUPP);
196	}
197
198	dmabuf = dma_buf_get(dmabuf_fd);
199	if (IS_ERR(dmabuf))
200		return ERR_CAST(dmabuf);
201
202	binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
203			       dev_to_node(&dev->dev));
204	if (!binding) {
205		err = -ENOMEM;
206		goto err_put_dmabuf;
207	}
208
209	binding->dev = dev;
210	xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
211
212	refcount_set(&binding->ref, 1);
213
214	mutex_init(&binding->lock);
215
216	binding->dmabuf = dmabuf;
217	binding->direction = direction;
218
219	binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
220	if (IS_ERR(binding->attachment)) {
221		err = PTR_ERR(binding->attachment);
222		NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
223		goto err_free_binding;
224	}
225
226	binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
227						       direction);
228	if (IS_ERR(binding->sgt)) {
229		err = PTR_ERR(binding->sgt);
230		NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
231		goto err_detach;
232	}
233
234	if (direction == DMA_TO_DEVICE) {
235		binding->tx_vec = kvmalloc_array(dmabuf->size / PAGE_SIZE,
236						 sizeof(struct net_iov *),
237						 GFP_KERNEL);
238		if (!binding->tx_vec) {
239			err = -ENOMEM;
240			goto err_unmap;
241		}
242	}
243
244	/* For simplicity we expect to make PAGE_SIZE allocations, but the
245	 * binding can be much more flexible than that. We may be able to
246	 * allocate MTU sized chunks here. Leave that for future work...
247	 */
248	binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
249					      dev_to_node(&dev->dev));
250	if (!binding->chunk_pool) {
251		err = -ENOMEM;
252		goto err_tx_vec;
253	}
254
255	virtual = 0;
256	for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
257		dma_addr_t dma_addr = sg_dma_address(sg);
258		struct dmabuf_genpool_chunk_owner *owner;
259		size_t len = sg_dma_len(sg);
260		struct net_iov *niov;
261
262		owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
263				     dev_to_node(&dev->dev));
264		if (!owner) {
265			err = -ENOMEM;
266			goto err_free_chunks;
267		}
268
269		owner->area.base_virtual = virtual;
270		owner->base_dma_addr = dma_addr;
271		owner->area.num_niovs = len / PAGE_SIZE;
272		owner->binding = binding;
273
274		err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
275					 dma_addr, len, dev_to_node(&dev->dev),
276					 owner);
277		if (err) {
278			kfree(owner);
279			err = -EINVAL;
280			goto err_free_chunks;
281		}
282
283		owner->area.niovs = kvmalloc_array(owner->area.num_niovs,
284						   sizeof(*owner->area.niovs),
285						   GFP_KERNEL);
286		if (!owner->area.niovs) {
287			err = -ENOMEM;
288			goto err_free_chunks;
289		}
290
291		for (i = 0; i < owner->area.num_niovs; i++) {
292			niov = &owner->area.niovs[i];
293			niov->type = NET_IOV_DMABUF;
294			niov->owner = &owner->area;
295			page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
296						      net_devmem_get_dma_addr(niov));
297			if (direction == DMA_TO_DEVICE)
298				binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
299		}
300
301		virtual += len;
302	}
303
304	err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
305			      binding, xa_limit_32b, &id_alloc_next,
306			      GFP_KERNEL);
307	if (err < 0)
308		goto err_free_chunks;
309
310	list_add(&binding->list, &priv->bindings);
311
312	return binding;
313
314err_free_chunks:
315	gen_pool_for_each_chunk(binding->chunk_pool,
316				net_devmem_dmabuf_free_chunk_owner, NULL);
317	gen_pool_destroy(binding->chunk_pool);
318err_tx_vec:
319	kvfree(binding->tx_vec);
320err_unmap:
321	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
322					  direction);
323err_detach:
324	dma_buf_detach(dmabuf, binding->attachment);
325err_free_binding:
326	kfree(binding);
327err_put_dmabuf:
328	dma_buf_put(dmabuf);
329	return ERR_PTR(err);
330}
331
332struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
333{
334	struct net_devmem_dmabuf_binding *binding;
335
336	rcu_read_lock();
337	binding = xa_load(&net_devmem_dmabuf_bindings, id);
338	if (binding) {
339		if (!net_devmem_dmabuf_binding_get(binding))
340			binding = NULL;
341	}
342	rcu_read_unlock();
343
344	return binding;
345}
346
347void net_devmem_get_net_iov(struct net_iov *niov)
348{
349	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
350}
351
352void net_devmem_put_net_iov(struct net_iov *niov)
353{
354	net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
355}
356
357struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
358							 unsigned int dmabuf_id)
359{
360	struct net_devmem_dmabuf_binding *binding;
361	struct net_device *dst_dev;
362	struct dst_entry *dst;
363	int err = 0;
364
365	binding = net_devmem_lookup_dmabuf(dmabuf_id);
366	if (!binding || !binding->tx_vec) {
367		err = -EINVAL;
368		goto out_err;
369	}
370
371	rcu_read_lock();
372	dst = __sk_dst_get(sk);
373	/* If dst is NULL (route expired), attempt to rebuild it. */
374	if (unlikely(!dst)) {
375		if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
376			err = -EHOSTUNREACH;
377			goto out_unlock;
378		}
379		dst = __sk_dst_get(sk);
380		if (unlikely(!dst)) {
381			err = -ENODEV;
382			goto out_unlock;
383		}
384	}
385
386	/* The dma-addrs in this binding are only reachable to the corresponding
387	 * net_device.
388	 */
389	dst_dev = dst_dev_rcu(dst);
390	if (unlikely(!dst_dev) ||
391	    unlikely(dst_dev != READ_ONCE(binding->dev))) {
392		err = -ENODEV;
393		goto out_unlock;
394	}
395
396	rcu_read_unlock();
397	return binding;
398
399out_unlock:
400	rcu_read_unlock();
401out_err:
402	if (binding)
403		net_devmem_dmabuf_binding_put(binding);
404
405	return ERR_PTR(err);
406}
407
408struct net_iov *
409net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
410		       size_t virt_addr, size_t *off, size_t *size)
411{
412	if (virt_addr >= binding->dmabuf->size)
413		return NULL;
414
415	*off = virt_addr % PAGE_SIZE;
416	*size = PAGE_SIZE - *off;
417
418	return binding->tx_vec[virt_addr / PAGE_SIZE];
419}
420
421/*** "Dmabuf devmem memory provider" ***/
422
423int mp_dmabuf_devmem_init(struct page_pool *pool)
424{
425	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
426
427	if (!binding)
428		return -EINVAL;
429
430	/* dma-buf dma addresses do not need and should not be used with
431	 * dma_sync_for_cpu/device. Force disable dma_sync.
432	 */
433	pool->dma_sync = false;
434	pool->dma_sync_for_cpu = false;
435
436	if (pool->p.order != 0)
437		return -E2BIG;
438
439	net_devmem_dmabuf_binding_get(binding);
440	return 0;
441}
442
443netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
444{
445	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
446	struct net_iov *niov;
447	netmem_ref netmem;
448
449	niov = net_devmem_alloc_dmabuf(binding);
450	if (!niov)
451		return 0;
452
453	netmem = net_iov_to_netmem(niov);
454
455	page_pool_set_pp_info(pool, netmem);
456
457	pool->pages_state_hold_cnt++;
458	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
459	return netmem;
460}
461
462void mp_dmabuf_devmem_destroy(struct page_pool *pool)
463{
464	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
465
466	net_devmem_dmabuf_binding_put(binding);
467}
468
469bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
470{
471	long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
472
473	if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
474		return false;
475
476	if (WARN_ON_ONCE(refcount != 1))
477		return false;
478
479	page_pool_clear_pp_info(netmem);
480
481	net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
482
483	/* We don't want the page pool put_page()ing our net_iovs. */
484	return false;
485}
486
487static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
488				    struct netdev_rx_queue *rxq)
489{
490	const struct net_devmem_dmabuf_binding *binding = mp_priv;
491	int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
492
493	return nla_put_u32(rsp, type, binding->id);
494}
495
496static void mp_dmabuf_devmem_uninstall(void *mp_priv,
497				       struct netdev_rx_queue *rxq)
498{
499	struct net_devmem_dmabuf_binding *binding = mp_priv;
500	struct netdev_rx_queue *bound_rxq;
501	unsigned long xa_idx;
502
503	xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
504		if (bound_rxq == rxq) {
505			xa_erase(&binding->bound_rxqs, xa_idx);
506			if (xa_empty(&binding->bound_rxqs)) {
507				mutex_lock(&binding->lock);
508				ASSERT_EXCLUSIVE_WRITER(binding->dev);
509				WRITE_ONCE(binding->dev, NULL);
510				mutex_unlock(&binding->lock);
511			}
512			break;
513		}
514	}
515}
516
517static const struct memory_provider_ops dmabuf_devmem_ops = {
518	.init			= mp_dmabuf_devmem_init,
519	.destroy		= mp_dmabuf_devmem_destroy,
520	.alloc_netmems		= mp_dmabuf_devmem_alloc_netmems,
521	.release_netmem		= mp_dmabuf_devmem_release_page,
522	.nl_fill		= mp_dmabuf_devmem_nl_fill,
523	.uninstall		= mp_dmabuf_devmem_uninstall,
524};