internal.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef __PACKET_INTERNAL_H__
  3#define __PACKET_INTERNAL_H__
  4
  5#include <linux/refcount.h>
  6
  7struct packet_mclist {
  8	struct packet_mclist	*next;
  9	int			ifindex;
 10	int			count;
 11	unsigned short		type;
 12	unsigned short		alen;
 13	unsigned char		addr[MAX_ADDR_LEN];
 14	struct list_head	remove_list;
 15};
 16
 17/* kbdq - kernel block descriptor queue */
 18struct tpacket_kbdq_core {
 19	struct pgv	*pkbdq;
 20	unsigned int	feature_req_word;
 21	unsigned int	hdrlen;
 22	unsigned char	reset_pending_on_curr_blk;
 23	unsigned short	kactive_blk_num;
 24	unsigned short	blk_sizeof_priv;
 25
 26	unsigned short  version;
 27
 28	char		*pkblk_start;
 29	char		*pkblk_end;
 30	int		kblk_size;
 31	unsigned int	max_frame_len;
 32	unsigned int	knum_blocks;
 33	uint64_t	knxt_seq_num;
 34	char		*prev;
 35	char		*nxt_offset;
 36
 37	struct sk_buff	*skb;
 38
 39	rwlock_t	blk_fill_in_prog_lock;
 40
 41	/* Default is set to 8ms */
 42#define DEFAULT_PRB_RETIRE_TOV	(8)
 43
 44	ktime_t		interval_ktime;
 45
 46	/* timer to retire an outstanding block */
 47	struct hrtimer  retire_blk_timer;
 48};
 49
 50struct pgv {
 51	char *buffer;
 52};
 53
 54struct packet_ring_buffer {
 55	struct pgv		*pg_vec;
 56
 57	unsigned int		head;
 58	unsigned int		frames_per_block;
 59	unsigned int		frame_size;
 60	unsigned int		frame_max;
 61
 62	unsigned int		pg_vec_order;
 63	unsigned int		pg_vec_pages;
 64	unsigned int		pg_vec_len;
 65
 66	unsigned int __percpu	*pending_refcnt;
 67
 68	union {
 69		unsigned long			*rx_owner_map;
 70		struct tpacket_kbdq_core	prb_bdqc;
 71	};
 72};
 73
 74extern struct mutex fanout_mutex;
 75#define PACKET_FANOUT_MAX	(1 << 16)
 76
 77struct packet_fanout {
 78	possible_net_t		net;
 79	unsigned int		num_members;
 80	u32			max_num_members;
 81	u16			id;
 82	u8			type;
 83	u8			flags;
 84	union {
 85		atomic_t		rr_cur;
 86		struct bpf_prog __rcu	*bpf_prog;
 87	};
 88	struct list_head	list;
 89	spinlock_t		lock;
 90	refcount_t		sk_ref;
 91	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
 92	struct sock	__rcu	*arr[] __counted_by(max_num_members);
 93};
 94
 95struct packet_rollover {
 96	int			sock;
 97	atomic_long_t		num;
 98	atomic_long_t		num_huge;
 99	atomic_long_t		num_failed;
100#define ROLLOVER_HLEN	(L1_CACHE_BYTES / sizeof(u32))
101	u32			history[ROLLOVER_HLEN] ____cacheline_aligned;
102} ____cacheline_aligned_in_smp;
103
104struct packet_sock {
105	/* struct sock has to be the first member of packet_sock */
106	struct sock		sk;
107	struct packet_fanout	*fanout;
108	union  tpacket_stats_u	stats;
109	struct packet_ring_buffer	rx_ring;
110	struct packet_ring_buffer	tx_ring;
111	int			copy_thresh;
112	spinlock_t		bind_lock;
113	struct mutex		pg_vec_lock;
114	unsigned long		flags;
115	int			ifindex;	/* bound device		*/
116	u8			vnet_hdr_sz;
117	__be16			num;
118	struct packet_rollover	*rollover;
119	struct packet_mclist	*mclist;
120	atomic_long_t		mapped;
121	enum tpacket_versions	tp_version;
122	unsigned int		tp_hdrlen;
123	unsigned int		tp_reserve;
124	unsigned int		tp_tstamp;
125	struct completion	skb_completion;
126	struct net_device __rcu	*cached_dev;
127	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
128	atomic_t		tp_drops ____cacheline_aligned_in_smp;
129};
130
131#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk)
132
133enum packet_sock_flags {
134	PACKET_SOCK_ORIGDEV,
135	PACKET_SOCK_AUXDATA,
136	PACKET_SOCK_TX_HAS_OFF,
137	PACKET_SOCK_TP_LOSS,
138	PACKET_SOCK_RUNNING,
139	PACKET_SOCK_PRESSURE,
140	PACKET_SOCK_QDISC_BYPASS,
141};
142
143static inline void packet_sock_flag_set(struct packet_sock *po,
144					enum packet_sock_flags flag,
145					bool val)
146{
147	if (val)
148		set_bit(flag, &po->flags);
149	else
150		clear_bit(flag, &po->flags);
151}
152
153static inline bool packet_sock_flag(const struct packet_sock *po,
154				    enum packet_sock_flags flag)
155{
156	return test_bit(flag, &po->flags);
157}
158
159#endif