zcrx.h

  1// SPDX-License-Identifier: GPL-2.0
  2#ifndef IOU_ZC_RX_H
  3#define IOU_ZC_RX_H
  4
  5#include <linux/io_uring_types.h>
  6#include <linux/dma-buf.h>
  7#include <linux/socket.h>
  8#include <net/page_pool/types.h>
  9#include <net/net_trackers.h>
 10
 11struct io_zcrx_mem {
 12	unsigned long			size;
 13	bool				is_dmabuf;
 14
 15	struct page			**pages;
 16	unsigned long			nr_folios;
 17	struct sg_table			page_sg_table;
 18	unsigned long			account_pages;
 19	struct sg_table			*sgt;
 20
 21	struct dma_buf_attachment	*attach;
 22	struct dma_buf			*dmabuf;
 23};
 24
 25struct io_zcrx_area {
 26	struct net_iov_area	nia;
 27	struct io_zcrx_ifq	*ifq;
 28	atomic_t		*user_refs;
 29
 30	bool			is_mapped;
 31	u16			area_id;
 32
 33	/* freelist */
 34	spinlock_t		freelist_lock ____cacheline_aligned_in_smp;
 35	u32			free_count;
 36	u32			*freelist;
 37
 38	struct io_zcrx_mem	mem;
 39};
 40
 41struct io_zcrx_ifq {
 42	struct io_zcrx_area		*area;
 43	unsigned			niov_shift;
 44	struct user_struct		*user;
 45	struct mm_struct		*mm_account;
 46
 47	spinlock_t			rq_lock ____cacheline_aligned_in_smp;
 48	struct io_uring			*rq_ring;
 49	struct io_uring_zcrx_rqe	*rqes;
 50	u32				cached_rq_head;
 51	u32				rq_entries;
 52
 53	u32				if_rxq;
 54	struct device			*dev;
 55	struct net_device		*netdev;
 56	netdevice_tracker		netdev_tracker;
 57	refcount_t			refs;
 58	/* counts userspace facing users like io_uring */
 59	refcount_t			user_refs;
 60
 61	/*
 62	 * Page pool and net configuration lock, can be taken deeper in the
 63	 * net stack.
 64	 */
 65	struct mutex			pp_lock;
 66	struct io_mapped_region		region;
 67};
 68
 69#if defined(CONFIG_IO_URING_ZCRX)
 70int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
 71int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
 72			 struct io_uring_zcrx_ifq_reg __user *arg);
 73void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
 74int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
 75		 struct socket *sock, unsigned int flags,
 76		 unsigned issue_flags, unsigned int *len);
 77struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
 78					    unsigned int id);
 79#else
 80static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
 81					struct io_uring_zcrx_ifq_reg __user *arg)
 82{
 83	return -EOPNOTSUPP;
 84}
 85static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
 86{
 87}
 88static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
 89			       struct socket *sock, unsigned int flags,
 90			       unsigned issue_flags, unsigned int *len)
 91{
 92	return -EOPNOTSUPP;
 93}
 94static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
 95							  unsigned int id)
 96{
 97	return NULL;
 98}
 99static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
100				void __user *arg, unsigned nr_arg)
101{
102	return -EOPNOTSUPP;
103}
104#endif
105
106int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
107int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
108
109#endif