nfnetlink_queue.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * This is a module which is used for queueing packets and communicating with
   4 * userspace via nfnetlink.
   5 *
   6 * (C) 2005 by Harald Welte <laforge@netfilter.org>
   7 * (C) 2007 by Patrick McHardy <kaber@trash.net>
   8 *
   9 * Based on the old ipv4-only ip_queue.c:
  10 * (C) 2000-2002 James Morris <jmorris@intercode.com.au>
  11 * (C) 2003-2005 Netfilter Core Team <coreteam@netfilter.org>
  12 */
  13
  14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  15
  16#include <linux/module.h>
  17#include <linux/skbuff.h>
  18#include <linux/init.h>
  19#include <linux/spinlock.h>
  20#include <linux/slab.h>
  21#include <linux/notifier.h>
  22#include <linux/netdevice.h>
  23#include <linux/netfilter.h>
  24#include <linux/proc_fs.h>
  25#include <linux/netfilter_ipv4.h>
  26#include <linux/netfilter_ipv6.h>
  27#include <linux/netfilter_bridge.h>
  28#include <linux/netfilter/nfnetlink.h>
  29#include <linux/netfilter/nfnetlink_queue.h>
  30#include <linux/netfilter/nf_conntrack_common.h>
  31#include <linux/list.h>
  32#include <linux/cgroup-defs.h>
  33#include <linux/rhashtable.h>
  34#include <linux/jhash.h>
  35#include <net/gso.h>
  36#include <net/sock.h>
  37#include <net/tcp_states.h>
  38#include <net/netfilter/nf_queue.h>
  39#include <net/netns/generic.h>
  40
  41#include <linux/atomic.h>
  42
  43#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
  44#include "../bridge/br_private.h"
  45#endif
  46
  47#if IS_ENABLED(CONFIG_NF_CONNTRACK)
  48#include <net/netfilter/nf_conntrack.h>
  49#endif
  50
  51#define NFQNL_QMAX_DEFAULT 1024
  52#define NFQNL_HASH_MIN     8
  53#define NFQNL_HASH_MAX     32768
  54
  55/* We're using struct nlattr which has 16bit nla_len. Note that nla_len
  56 * includes the header length. Thus, the maximum packet length that we
  57 * support is 65531 bytes. We send truncated packets if the specified length
  58 * is larger than that.  Userspace can check for presence of NFQA_CAP_LEN
  59 * attribute to detect truncation.
  60 */
  61#define NFQNL_MAX_COPY_RANGE (0xffff - NLA_HDRLEN)
  62
  63struct nfqnl_instance {
  64	struct hlist_node hlist;		/* global list of queues */
  65	struct rhashtable nfqnl_packet_map;
  66	struct rcu_work	rwork;
  67
  68	u32 peer_portid;
  69	unsigned int queue_maxlen;
  70	unsigned int copy_range;
  71	unsigned int queue_dropped;
  72	unsigned int queue_user_dropped;
  73
  74
  75	u_int16_t queue_num;			/* number of this queue */
  76	u_int8_t copy_mode;
  77	u_int32_t flags;			/* Set using NFQA_CFG_FLAGS */
  78/*
  79 * Following fields are dirtied for each queued packet,
  80 * keep them in same cache line if possible.
  81 */
  82	spinlock_t	lock	____cacheline_aligned_in_smp;
  83	unsigned int	queue_total;
  84	unsigned int	id_sequence;		/* 'sequence' of pkt ids */
  85	struct list_head queue_list;		/* packets in queue */
  86};
  87
  88typedef int (*nfqnl_cmpfn)(struct nf_queue_entry *, unsigned long);
  89
  90static struct workqueue_struct *nfq_cleanup_wq __read_mostly;
  91static unsigned int nfnl_queue_net_id __read_mostly;
  92
  93#define INSTANCE_BUCKETS	16
  94struct nfnl_queue_net {
  95	spinlock_t instances_lock;
  96	struct hlist_head instance_table[INSTANCE_BUCKETS];
  97};
  98
  99static struct nfnl_queue_net *nfnl_queue_pernet(struct net *net)
 100{
 101	return net_generic(net, nfnl_queue_net_id);
 102}
 103
 104static inline u_int8_t instance_hashfn(u_int16_t queue_num)
 105{
 106	return ((queue_num >> 8) ^ queue_num) % INSTANCE_BUCKETS;
 107}
 108
 109static const struct rhashtable_params nfqnl_rhashtable_params = {
 110	.head_offset = offsetof(struct nf_queue_entry, hash_node),
 111	.key_offset = offsetof(struct nf_queue_entry, id),
 112	.key_len = sizeof(u32),
 113	.automatic_shrinking = true,
 114	.min_size = NFQNL_HASH_MIN,
 115	.max_size = NFQNL_HASH_MAX,
 116};
 117
 118static struct nfqnl_instance *
 119instance_lookup(struct nfnl_queue_net *q, u_int16_t queue_num)
 120{
 121	struct hlist_head *head;
 122	struct nfqnl_instance *inst;
 123
 124	head = &q->instance_table[instance_hashfn(queue_num)];
 125	hlist_for_each_entry_rcu(inst, head, hlist) {
 126		if (inst->queue_num == queue_num)
 127			return inst;
 128	}
 129	return NULL;
 130}
 131
 132static struct nfqnl_instance *
 133instance_create(struct nfnl_queue_net *q, u_int16_t queue_num, u32 portid)
 134{
 135	struct nfqnl_instance *inst;
 136	unsigned int h;
 137	int err;
 138
 139	inst = kzalloc(sizeof(*inst), GFP_KERNEL_ACCOUNT);
 140	if (!inst)
 141		return ERR_PTR(-ENOMEM);
 142
 143	inst->queue_num = queue_num;
 144	inst->peer_portid = portid;
 145	inst->queue_maxlen = NFQNL_QMAX_DEFAULT;
 146	inst->copy_range = NFQNL_MAX_COPY_RANGE;
 147	inst->copy_mode = NFQNL_COPY_NONE;
 148	spin_lock_init(&inst->lock);
 149	INIT_LIST_HEAD(&inst->queue_list);
 150
 151	err = rhashtable_init(&inst->nfqnl_packet_map, &nfqnl_rhashtable_params);
 152	if (err < 0)
 153		goto out_free;
 154
 155	spin_lock(&q->instances_lock);
 156	if (instance_lookup(q, queue_num)) {
 157		err = -EEXIST;
 158		goto out_unlock;
 159	}
 160
 161	if (!try_module_get(THIS_MODULE)) {
 162		err = -EAGAIN;
 163		goto out_unlock;
 164	}
 165
 166	h = instance_hashfn(queue_num);
 167	hlist_add_head_rcu(&inst->hlist, &q->instance_table[h]);
 168
 169	spin_unlock(&q->instances_lock);
 170
 171	return inst;
 172
 173out_unlock:
 174	spin_unlock(&q->instances_lock);
 175	rhashtable_destroy(&inst->nfqnl_packet_map);
 176out_free:
 177	kfree(inst);
 178	return ERR_PTR(err);
 179}
 180
 181static void nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn,
 182			unsigned long data);
 183
 184static void instance_destroy_work(struct work_struct *work)
 185{
 186	struct nfqnl_instance *inst;
 187
 188	inst = container_of(to_rcu_work(work), struct nfqnl_instance,
 189			    rwork);
 190	rcu_read_lock();
 191	nfqnl_flush(inst, NULL, 0);
 192	rcu_read_unlock();
 193
 194	rhashtable_destroy(&inst->nfqnl_packet_map);
 195
 196	kfree(inst);
 197	module_put(THIS_MODULE);
 198}
 199
 200static void
 201__instance_destroy(struct nfqnl_instance *inst)
 202{
 203	hlist_del_rcu(&inst->hlist);
 204
 205	INIT_RCU_WORK(&inst->rwork, instance_destroy_work);
 206	queue_rcu_work(nfq_cleanup_wq, &inst->rwork);
 207}
 208
 209static void
 210instance_destroy(struct nfnl_queue_net *q, struct nfqnl_instance *inst)
 211{
 212	spin_lock(&q->instances_lock);
 213	__instance_destroy(inst);
 214	spin_unlock(&q->instances_lock);
 215}
 216
 217static int
 218__enqueue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry)
 219{
 220	int err;
 221
 222	err = rhashtable_insert_fast(&queue->nfqnl_packet_map, &entry->hash_node,
 223				     nfqnl_rhashtable_params);
 224	if (unlikely(err))
 225		return err;
 226
 227	list_add_tail(&entry->list, &queue->queue_list);
 228	queue->queue_total++;
 229
 230	return 0;
 231}
 232
 233static void
 234__dequeue_entry(struct nfqnl_instance *queue, struct nf_queue_entry *entry)
 235{
 236	rhashtable_remove_fast(&queue->nfqnl_packet_map, &entry->hash_node,
 237			       nfqnl_rhashtable_params);
 238	list_del(&entry->list);
 239	queue->queue_total--;
 240}
 241
 242static struct nf_queue_entry *
 243find_dequeue_entry(struct nfqnl_instance *queue, unsigned int id)
 244{
 245	struct nf_queue_entry *entry;
 246
 247	spin_lock_bh(&queue->lock);
 248	entry = rhashtable_lookup_fast(&queue->nfqnl_packet_map, &id,
 249				       nfqnl_rhashtable_params);
 250
 251	if (entry)
 252		__dequeue_entry(queue, entry);
 253
 254	spin_unlock_bh(&queue->lock);
 255
 256	return entry;
 257}
 258
 259static unsigned int nf_iterate(struct sk_buff *skb,
 260			       struct nf_hook_state *state,
 261			       const struct nf_hook_entries *hooks,
 262			       unsigned int *index)
 263{
 264	const struct nf_hook_entry *hook;
 265	unsigned int verdict, i = *index;
 266
 267	while (i < hooks->num_hook_entries) {
 268		hook = &hooks->hooks[i];
 269repeat:
 270		verdict = nf_hook_entry_hookfn(hook, skb, state);
 271		if (verdict != NF_ACCEPT) {
 272			*index = i;
 273			if (verdict != NF_REPEAT)
 274				return verdict;
 275			goto repeat;
 276		}
 277		i++;
 278	}
 279
 280	*index = i;
 281	return NF_ACCEPT;
 282}
 283
 284static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum)
 285{
 286	switch (pf) {
 287#ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
 288	case NFPROTO_BRIDGE:
 289		return rcu_dereference(net->nf.hooks_bridge[hooknum]);
 290#endif
 291	case NFPROTO_IPV4:
 292		return rcu_dereference(net->nf.hooks_ipv4[hooknum]);
 293	case NFPROTO_IPV6:
 294		return rcu_dereference(net->nf.hooks_ipv6[hooknum]);
 295	default:
 296		WARN_ON_ONCE(1);
 297		return NULL;
 298	}
 299
 300	return NULL;
 301}
 302
 303static int nf_ip_reroute(struct sk_buff *skb, const struct nf_queue_entry *entry)
 304{
 305#ifdef CONFIG_INET
 306	const struct ip_rt_info *rt_info = nf_queue_entry_reroute(entry);
 307
 308	if (entry->state.hook == NF_INET_LOCAL_OUT) {
 309		const struct iphdr *iph = ip_hdr(skb);
 310
 311		if (!(iph->tos == rt_info->tos &&
 312		      skb->mark == rt_info->mark &&
 313		      iph->daddr == rt_info->daddr &&
 314		      iph->saddr == rt_info->saddr))
 315			return ip_route_me_harder(entry->state.net, entry->state.sk,
 316						  skb, RTN_UNSPEC);
 317	}
 318#endif
 319	return 0;
 320}
 321
 322static int nf_reroute(struct sk_buff *skb, struct nf_queue_entry *entry)
 323{
 324	const struct nf_ipv6_ops *v6ops;
 325	int ret = 0;
 326
 327	switch (entry->state.pf) {
 328	case AF_INET:
 329		ret = nf_ip_reroute(skb, entry);
 330		break;
 331	case AF_INET6:
 332		v6ops = rcu_dereference(nf_ipv6_ops);
 333		if (v6ops)
 334			ret = v6ops->reroute(skb, entry);
 335		break;
 336	}
 337	return ret;
 338}
 339
 340/* caller must hold rcu read-side lock */
 341static void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
 342{
 343	const struct nf_hook_entry *hook_entry;
 344	const struct nf_hook_entries *hooks;
 345	struct sk_buff *skb = entry->skb;
 346	const struct net *net;
 347	unsigned int i;
 348	int err;
 349	u8 pf;
 350
 351	net = entry->state.net;
 352	pf = entry->state.pf;
 353
 354	hooks = nf_hook_entries_head(net, pf, entry->state.hook);
 355
 356	i = entry->hook_index;
 357	if (!hooks || i >= hooks->num_hook_entries) {
 358		kfree_skb_reason(skb, SKB_DROP_REASON_NETFILTER_DROP);
 359		nf_queue_entry_free(entry);
 360		return;
 361	}
 362
 363	hook_entry = &hooks->hooks[i];
 364
 365	/* Continue traversal iff userspace said ok... */
 366	if (verdict == NF_REPEAT)
 367		verdict = nf_hook_entry_hookfn(hook_entry, skb, &entry->state);
 368
 369	if (verdict == NF_ACCEPT) {
 370		if (nf_reroute(skb, entry) < 0)
 371			verdict = NF_DROP;
 372	}
 373
 374	if (verdict == NF_ACCEPT) {
 375next_hook:
 376		++i;
 377		verdict = nf_iterate(skb, &entry->state, hooks, &i);
 378	}
 379
 380	switch (verdict & NF_VERDICT_MASK) {
 381	case NF_ACCEPT:
 382	case NF_STOP:
 383		local_bh_disable();
 384		entry->state.okfn(entry->state.net, entry->state.sk, skb);
 385		local_bh_enable();
 386		break;
 387	case NF_QUEUE:
 388		err = nf_queue(skb, &entry->state, i, verdict);
 389		if (err == 1)
 390			goto next_hook;
 391		break;
 392	case NF_STOLEN:
 393		break;
 394	default:
 395		kfree_skb(skb);
 396	}
 397
 398	nf_queue_entry_free(entry);
 399}
 400
 401/* return true if the entry has an unconfirmed conntrack attached that isn't owned by us
 402 * exclusively.
 403 */
 404static bool nf_ct_drop_unconfirmed(const struct nf_queue_entry *entry, bool *is_unconfirmed)
 405{
 406#if IS_ENABLED(CONFIG_NF_CONNTRACK)
 407	struct nf_conn *ct = (void *)skb_nfct(entry->skb);
 408
 409	if (!ct || nf_ct_is_confirmed(ct))
 410		return false;
 411
 412	if (is_unconfirmed)
 413		*is_unconfirmed = true;
 414
 415	/* in some cases skb_clone() can occur after initial conntrack
 416	 * pickup, but conntrack assumes exclusive skb->_nfct ownership for
 417	 * unconfirmed entries.
 418	 *
 419	 * This happens for br_netfilter and with ip multicast routing.
 420	 * This can't be solved with serialization here because one clone
 421	 * could have been queued for local delivery or could be transmitted
 422	 * in parallel on another CPU.
 423	 */
 424	return refcount_read(&ct->ct_general.use) > 1;
 425#endif
 426	return false;
 427}
 428
 429static void nfqnl_reinject(struct nf_queue_entry *entry, unsigned int verdict)
 430{
 431	const struct nf_ct_hook *ct_hook;
 432
 433	if (verdict == NF_ACCEPT ||
 434	    verdict == NF_REPEAT ||
 435	    verdict == NF_STOP) {
 436		unsigned int ct_verdict = verdict;
 437
 438		rcu_read_lock();
 439		ct_hook = rcu_dereference(nf_ct_hook);
 440		if (ct_hook)
 441			ct_verdict = ct_hook->update(entry->state.net, entry->skb);
 442		rcu_read_unlock();
 443
 444		switch (ct_verdict & NF_VERDICT_MASK) {
 445		case NF_ACCEPT:
 446			/* follow userspace verdict, could be REPEAT */
 447			break;
 448		case NF_STOLEN:
 449			nf_queue_entry_free(entry);
 450			return;
 451		default:
 452			verdict = ct_verdict & NF_VERDICT_MASK;
 453			break;
 454		}
 455	}
 456
 457	if (verdict != NF_DROP && entry->nf_ct_is_unconfirmed) {
 458		/* If first queued segment was already reinjected then
 459		 * there is a good chance the ct entry is now confirmed.
 460		 *
 461		 * Handle the rare cases:
 462		 *  - out-of-order verdict
 463		 *  - threaded userspace reinjecting in parallel
 464		 *  - first segment was dropped
 465		 *
 466		 * In all of those cases we can't handle this packet
 467		 * because we can't be sure that another CPU won't modify
 468		 * nf_conn->ext in parallel which isn't allowed.
 469		 */
 470		if (nf_ct_drop_unconfirmed(entry, NULL))
 471			verdict = NF_DROP;
 472	}
 473
 474	nf_reinject(entry, verdict);
 475}
 476
 477static void
 478nfqnl_flush(struct nfqnl_instance *queue, nfqnl_cmpfn cmpfn, unsigned long data)
 479{
 480	struct nf_queue_entry *entry, *next;
 481
 482	spin_lock_bh(&queue->lock);
 483	list_for_each_entry_safe(entry, next, &queue->queue_list, list) {
 484		if (!cmpfn || cmpfn(entry, data)) {
 485			__dequeue_entry(queue, entry);
 486			nfqnl_reinject(entry, NF_DROP);
 487		}
 488	}
 489	spin_unlock_bh(&queue->lock);
 490}
 491
 492static int
 493nfqnl_put_packet_info(struct sk_buff *nlskb, struct sk_buff *packet,
 494		      bool csum_verify)
 495{
 496	__u32 flags = 0;
 497
 498	if (packet->ip_summed == CHECKSUM_PARTIAL)
 499		flags = NFQA_SKB_CSUMNOTREADY;
 500	else if (csum_verify)
 501		flags = NFQA_SKB_CSUM_NOTVERIFIED;
 502
 503	if (skb_is_gso(packet))
 504		flags |= NFQA_SKB_GSO;
 505
 506	return flags ? nla_put_be32(nlskb, NFQA_SKB_INFO, htonl(flags)) : 0;
 507}
 508
 509static int nfqnl_put_sk_uidgid(struct sk_buff *skb, struct sock *sk)
 510{
 511	const struct cred *cred;
 512
 513	if (!sk_fullsock(sk))
 514		return 0;
 515
 516	read_lock_bh(&sk->sk_callback_lock);
 517	if (sk->sk_socket && sk->sk_socket->file) {
 518		cred = sk->sk_socket->file->f_cred;
 519		if (nla_put_be32(skb, NFQA_UID,
 520		    htonl(from_kuid_munged(&init_user_ns, cred->fsuid))))
 521			goto nla_put_failure;
 522		if (nla_put_be32(skb, NFQA_GID,
 523		    htonl(from_kgid_munged(&init_user_ns, cred->fsgid))))
 524			goto nla_put_failure;
 525	}
 526	read_unlock_bh(&sk->sk_callback_lock);
 527	return 0;
 528
 529nla_put_failure:
 530	read_unlock_bh(&sk->sk_callback_lock);
 531	return -1;
 532}
 533
 534static int nfqnl_put_sk_classid(struct sk_buff *skb, struct sock *sk)
 535{
 536#if IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)
 537	if (sk && sk_fullsock(sk)) {
 538		u32 classid = sock_cgroup_classid(&sk->sk_cgrp_data);
 539
 540		if (classid && nla_put_be32(skb, NFQA_CGROUP_CLASSID, htonl(classid)))
 541			return -1;
 542	}
 543#endif
 544	return 0;
 545}
 546
 547static int nfqnl_get_sk_secctx(struct sk_buff *skb, struct lsm_context *ctx)
 548{
 549	int seclen = 0;
 550#if IS_ENABLED(CONFIG_NETWORK_SECMARK)
 551
 552	if (!skb || !sk_fullsock(skb->sk))
 553		return 0;
 554
 555	read_lock_bh(&skb->sk->sk_callback_lock);
 556
 557	if (skb->secmark)
 558		seclen = security_secid_to_secctx(skb->secmark, ctx);
 559	read_unlock_bh(&skb->sk->sk_callback_lock);
 560#endif
 561	return seclen;
 562}
 563
 564static u32 nfqnl_get_bridge_size(struct nf_queue_entry *entry)
 565{
 566	struct sk_buff *entskb = entry->skb;
 567	u32 nlalen = 0;
 568
 569	if (entry->state.pf != PF_BRIDGE || !skb_mac_header_was_set(entskb))
 570		return 0;
 571
 572	if (skb_vlan_tag_present(entskb))
 573		nlalen += nla_total_size(nla_total_size(sizeof(__be16)) +
 574					 nla_total_size(sizeof(__be16)));
 575
 576	if (entskb->network_header > entskb->mac_header)
 577		nlalen += nla_total_size((entskb->network_header -
 578					  entskb->mac_header));
 579
 580	return nlalen;
 581}
 582
 583static int nfqnl_put_bridge(struct nf_queue_entry *entry, struct sk_buff *skb)
 584{
 585	struct sk_buff *entskb = entry->skb;
 586
 587	if (entry->state.pf != PF_BRIDGE || !skb_mac_header_was_set(entskb))
 588		return 0;
 589
 590	if (skb_vlan_tag_present(entskb)) {
 591		struct nlattr *nest;
 592
 593		nest = nla_nest_start(skb, NFQA_VLAN);
 594		if (!nest)
 595			goto nla_put_failure;
 596
 597		if (nla_put_be16(skb, NFQA_VLAN_TCI, htons(entskb->vlan_tci)) ||
 598		    nla_put_be16(skb, NFQA_VLAN_PROTO, entskb->vlan_proto))
 599			goto nla_put_failure;
 600
 601		nla_nest_end(skb, nest);
 602	}
 603
 604	if (entskb->mac_header < entskb->network_header) {
 605		int len = (int)(entskb->network_header - entskb->mac_header);
 606
 607		if (nla_put(skb, NFQA_L2HDR, len, skb_mac_header(entskb)))
 608			goto nla_put_failure;
 609	}
 610
 611	return 0;
 612
 613nla_put_failure:
 614	return -1;
 615}
 616
 617static int nf_queue_checksum_help(struct sk_buff *entskb)
 618{
 619	if (skb_csum_is_sctp(entskb))
 620		return skb_crc32c_csum_help(entskb);
 621
 622	return skb_checksum_help(entskb);
 623}
 624
 625static struct sk_buff *
 626nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue,
 627			   struct nf_queue_entry *entry,
 628			   __be32 **packet_id_ptr)
 629{
 630	size_t size;
 631	size_t data_len = 0, cap_len = 0;
 632	unsigned int hlen = 0;
 633	struct sk_buff *skb;
 634	struct nlattr *nla;
 635	struct nfqnl_msg_packet_hdr *pmsg;
 636	struct nlmsghdr *nlh;
 637	struct sk_buff *entskb = entry->skb;
 638	struct net_device *indev;
 639	struct net_device *outdev;
 640	struct nf_conn *ct = NULL;
 641	enum ip_conntrack_info ctinfo = 0;
 642	const struct nfnl_ct_hook *nfnl_ct;
 643	bool csum_verify;
 644	struct lsm_context ctx = { NULL, 0, 0 };
 645	int seclen = 0;
 646	ktime_t tstamp;
 647
 648	size = nlmsg_total_size(sizeof(struct nfgenmsg))
 649		+ nla_total_size(sizeof(struct nfqnl_msg_packet_hdr))
 650		+ nla_total_size(sizeof(u_int32_t))	/* ifindex */
 651		+ nla_total_size(sizeof(u_int32_t))	/* ifindex */
 652#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
 653		+ nla_total_size(sizeof(u_int32_t))	/* ifindex */
 654		+ nla_total_size(sizeof(u_int32_t))	/* ifindex */
 655#endif
 656		+ nla_total_size(sizeof(u_int32_t))	/* mark */
 657		+ nla_total_size(sizeof(u_int32_t))	/* priority */
 658		+ nla_total_size(sizeof(struct nfqnl_msg_packet_hw))
 659		+ nla_total_size(sizeof(u_int32_t))	/* skbinfo */
 660#if IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)
 661		+ nla_total_size(sizeof(u_int32_t))	/* classid */
 662#endif
 663		+ nla_total_size(sizeof(u_int32_t));	/* cap_len */
 664
 665	tstamp = skb_tstamp_cond(entskb, false);
 666	if (tstamp)
 667		size += nla_total_size(sizeof(struct nfqnl_msg_packet_timestamp));
 668
 669	size += nfqnl_get_bridge_size(entry);
 670
 671	if (entry->state.hook <= NF_INET_FORWARD ||
 672	   (entry->state.hook == NF_INET_POST_ROUTING && entskb->sk == NULL))
 673		csum_verify = !skb_csum_unnecessary(entskb);
 674	else
 675		csum_verify = false;
 676
 677	outdev = entry->state.out;
 678
 679	switch ((enum nfqnl_config_mode)READ_ONCE(queue->copy_mode)) {
 680	case NFQNL_COPY_META:
 681	case NFQNL_COPY_NONE:
 682		break;
 683
 684	case NFQNL_COPY_PACKET:
 685		if (!(queue->flags & NFQA_CFG_F_GSO) &&
 686		    entskb->ip_summed == CHECKSUM_PARTIAL &&
 687		    nf_queue_checksum_help(entskb))
 688			return NULL;
 689
 690		data_len = READ_ONCE(queue->copy_range);
 691		if (data_len > entskb->len)
 692			data_len = entskb->len;
 693
 694		hlen = skb_zerocopy_headlen(entskb);
 695		hlen = min_t(unsigned int, hlen, data_len);
 696		size += sizeof(struct nlattr) + hlen;
 697		cap_len = entskb->len;
 698		break;
 699	}
 700
 701	nfnl_ct = rcu_dereference(nfnl_ct_hook);
 702
 703#if IS_ENABLED(CONFIG_NF_CONNTRACK)
 704	if (queue->flags & NFQA_CFG_F_CONNTRACK) {
 705		if (nfnl_ct != NULL) {
 706			ct = nf_ct_get(entskb, &ctinfo);
 707			if (ct != NULL)
 708				size += nfnl_ct->build_size(ct);
 709		}
 710	}
 711#endif
 712
 713	if (queue->flags & NFQA_CFG_F_UID_GID) {
 714		size += (nla_total_size(sizeof(u_int32_t))	/* uid */
 715			+ nla_total_size(sizeof(u_int32_t)));	/* gid */
 716	}
 717
 718	if ((queue->flags & NFQA_CFG_F_SECCTX) && entskb->sk) {
 719		seclen = nfqnl_get_sk_secctx(entskb, &ctx);
 720		if (seclen < 0)
 721			return NULL;
 722		if (seclen)
 723			size += nla_total_size(seclen);
 724	}
 725
 726	skb = alloc_skb(size, GFP_ATOMIC);
 727	if (!skb) {
 728		skb_tx_error(entskb);
 729		goto nlmsg_failure;
 730	}
 731
 732	nlh = nfnl_msg_put(skb, 0, 0,
 733			   nfnl_msg_type(NFNL_SUBSYS_QUEUE, NFQNL_MSG_PACKET),
 734			   0, entry->state.pf, NFNETLINK_V0,
 735			   htons(queue->queue_num));
 736	if (!nlh) {
 737		skb_tx_error(entskb);
 738		kfree_skb(skb);
 739		goto nlmsg_failure;
 740	}
 741
 742	nla = __nla_reserve(skb, NFQA_PACKET_HDR, sizeof(*pmsg));
 743	pmsg = nla_data(nla);
 744	pmsg->hw_protocol	= entskb->protocol;
 745	pmsg->hook		= entry->state.hook;
 746	*packet_id_ptr		= &pmsg->packet_id;
 747
 748	indev = entry->state.in;
 749	if (indev) {
 750#if !IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
 751		if (nla_put_be32(skb, NFQA_IFINDEX_INDEV, htonl(indev->ifindex)))
 752			goto nla_put_failure;
 753#else
 754		if (entry->state.pf == PF_BRIDGE) {
 755			/* Case 1: indev is physical input device, we need to
 756			 * look for bridge group (when called from
 757			 * netfilter_bridge) */
 758			if (nla_put_be32(skb, NFQA_IFINDEX_PHYSINDEV,
 759					 htonl(indev->ifindex)) ||
 760			/* this is the bridge group "brX" */
 761			/* rcu_read_lock()ed by __nf_queue */
 762			    nla_put_be32(skb, NFQA_IFINDEX_INDEV,
 763					 htonl(br_port_get_rcu(indev)->br->dev->ifindex)))
 764				goto nla_put_failure;
 765		} else {
 766			int physinif;
 767
 768			/* Case 2: indev is bridge group, we need to look for
 769			 * physical device (when called from ipv4) */
 770			if (nla_put_be32(skb, NFQA_IFINDEX_INDEV,
 771					 htonl(indev->ifindex)))
 772				goto nla_put_failure;
 773
 774			physinif = nf_bridge_get_physinif(entskb);
 775			if (physinif &&
 776			    nla_put_be32(skb, NFQA_IFINDEX_PHYSINDEV,
 777					 htonl(physinif)))
 778				goto nla_put_failure;
 779		}
 780#endif
 781	}
 782
 783	if (outdev) {
 784#if !IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
 785		if (nla_put_be32(skb, NFQA_IFINDEX_OUTDEV, htonl(outdev->ifindex)))
 786			goto nla_put_failure;
 787#else
 788		if (entry->state.pf == PF_BRIDGE) {
 789			/* Case 1: outdev is physical output device, we need to
 790			 * look for bridge group (when called from
 791			 * netfilter_bridge) */
 792			if (nla_put_be32(skb, NFQA_IFINDEX_PHYSOUTDEV,
 793					 htonl(outdev->ifindex)) ||
 794			/* this is the bridge group "brX" */
 795			/* rcu_read_lock()ed by __nf_queue */
 796			    nla_put_be32(skb, NFQA_IFINDEX_OUTDEV,
 797					 htonl(br_port_get_rcu(outdev)->br->dev->ifindex)))
 798				goto nla_put_failure;
 799		} else {
 800			int physoutif;
 801
 802			/* Case 2: outdev is bridge group, we need to look for
 803			 * physical output device (when called from ipv4) */
 804			if (nla_put_be32(skb, NFQA_IFINDEX_OUTDEV,
 805					 htonl(outdev->ifindex)))
 806				goto nla_put_failure;
 807
 808			physoutif = nf_bridge_get_physoutif(entskb);
 809			if (physoutif &&
 810			    nla_put_be32(skb, NFQA_IFINDEX_PHYSOUTDEV,
 811					 htonl(physoutif)))
 812				goto nla_put_failure;
 813		}
 814#endif
 815	}
 816
 817	if (entskb->mark &&
 818	    nla_put_be32(skb, NFQA_MARK, htonl(entskb->mark)))
 819		goto nla_put_failure;
 820
 821	if (entskb->priority &&
 822	    nla_put_be32(skb, NFQA_PRIORITY, htonl(entskb->priority)))
 823		goto nla_put_failure;
 824
 825	if (indev && entskb->dev &&
 826	    skb_mac_header_was_set(entskb) &&
 827	    skb_mac_header_len(entskb) != 0) {
 828		struct nfqnl_msg_packet_hw phw;
 829		int len;
 830
 831		memset(&phw, 0, sizeof(phw));
 832		len = dev_parse_header(entskb, phw.hw_addr);
 833		if (len) {
 834			phw.hw_addrlen = htons(len);
 835			if (nla_put(skb, NFQA_HWADDR, sizeof(phw), &phw))
 836				goto nla_put_failure;
 837		}
 838	}
 839
 840	if (nfqnl_put_bridge(entry, skb) < 0)
 841		goto nla_put_failure;
 842
 843	if (entry->state.hook <= NF_INET_FORWARD && tstamp) {
 844		struct nfqnl_msg_packet_timestamp ts;
 845		struct timespec64 kts = ktime_to_timespec64(tstamp);
 846
 847		ts.sec = cpu_to_be64(kts.tv_sec);
 848		ts.usec = cpu_to_be64(kts.tv_nsec / NSEC_PER_USEC);
 849
 850		if (nla_put(skb, NFQA_TIMESTAMP, sizeof(ts), &ts))
 851			goto nla_put_failure;
 852	}
 853
 854	if ((queue->flags & NFQA_CFG_F_UID_GID) && entskb->sk &&
 855	    nfqnl_put_sk_uidgid(skb, entskb->sk) < 0)
 856		goto nla_put_failure;
 857
 858	if (nfqnl_put_sk_classid(skb, entskb->sk) < 0)
 859		goto nla_put_failure;
 860
 861	if (seclen > 0 && nla_put(skb, NFQA_SECCTX, ctx.len, ctx.context))
 862		goto nla_put_failure;
 863
 864	if (ct && nfnl_ct->build(skb, ct, ctinfo, NFQA_CT, NFQA_CT_INFO) < 0)
 865		goto nla_put_failure;
 866
 867	if (cap_len > data_len &&
 868	    nla_put_be32(skb, NFQA_CAP_LEN, htonl(cap_len)))
 869		goto nla_put_failure;
 870
 871	if (nfqnl_put_packet_info(skb, entskb, csum_verify))
 872		goto nla_put_failure;
 873
 874	if (data_len) {
 875		struct nlattr *nla;
 876
 877		if (skb_tailroom(skb) < sizeof(*nla) + hlen)
 878			goto nla_put_failure;
 879
 880		nla = skb_put(skb, sizeof(*nla));
 881		nla->nla_type = NFQA_PAYLOAD;
 882		nla->nla_len = nla_attr_size(data_len);
 883
 884		if (skb_zerocopy(skb, entskb, data_len, hlen))
 885			goto nla_put_failure;
 886	}
 887
 888	nlh->nlmsg_len = skb->len;
 889	if (seclen >= 0)
 890		security_release_secctx(&ctx);
 891	return skb;
 892
 893nla_put_failure:
 894	skb_tx_error(entskb);
 895	kfree_skb(skb);
 896	net_err_ratelimited("nf_queue: error creating packet message\n");
 897nlmsg_failure:
 898	if (seclen >= 0)
 899		security_release_secctx(&ctx);
 900	return NULL;
 901}
 902
 903static int
 904__nfqnl_enqueue_packet(struct net *net, struct nfqnl_instance *queue,
 905			struct nf_queue_entry *entry)
 906{
 907	struct sk_buff *nskb;
 908	int err = -ENOBUFS;
 909	__be32 *packet_id_ptr;
 910	int failopen = 0;
 911
 912	nskb = nfqnl_build_packet_message(net, queue, entry, &packet_id_ptr);
 913	if (nskb == NULL) {
 914		err = -ENOMEM;
 915		goto err_out;
 916	}
 917	spin_lock_bh(&queue->lock);
 918
 919	if (queue->queue_total >= queue->queue_maxlen)
 920		goto err_out_queue_drop;
 921
 922	entry->id = ++queue->id_sequence;
 923	*packet_id_ptr = htonl(entry->id);
 924
 925	/* Insert into hash BEFORE unicast. If failure don't send to userspace. */
 926	err = __enqueue_entry(queue, entry);
 927	if (unlikely(err))
 928		goto err_out_queue_drop;
 929
 930	/* nfnetlink_unicast will either free the nskb or add it to a socket */
 931	err = nfnetlink_unicast(nskb, net, queue->peer_portid);
 932	if (err < 0) {
 933		/* Unicast failed - remove entry we just inserted */
 934		__dequeue_entry(queue, entry);
 935
 936		if (queue->flags & NFQA_CFG_F_FAIL_OPEN) {
 937			failopen = 1;
 938			err = 0;
 939		} else {
 940			queue->queue_user_dropped++;
 941		}
 942		goto err_out_unlock;
 943	}
 944
 945	spin_unlock_bh(&queue->lock);
 946	return 0;
 947
 948err_out_queue_drop:
 949	if (queue->flags & NFQA_CFG_F_FAIL_OPEN) {
 950		failopen = 1;
 951		err = 0;
 952	} else {
 953		queue->queue_dropped++;
 954
 955		if (queue->queue_total >= queue->queue_maxlen)
 956			net_warn_ratelimited("nf_queue: full at %d entries, dropping packets(s)\n",
 957					     queue->queue_total);
 958		else
 959			net_warn_ratelimited("nf_queue: hash insert failed: %d\n", err);
 960	}
 961	kfree_skb(nskb);
 962err_out_unlock:
 963	spin_unlock_bh(&queue->lock);
 964	if (failopen)
 965		nfqnl_reinject(entry, NF_ACCEPT);
 966err_out:
 967	return err;
 968}
 969
 970static struct nf_queue_entry *
 971nf_queue_entry_dup(struct nf_queue_entry *e)
 972{
 973	struct nf_queue_entry *entry = kmemdup(e, e->size, GFP_ATOMIC);
 974
 975	if (!entry)
 976		return NULL;
 977
 978	if (nf_queue_entry_get_refs(entry))
 979		return entry;
 980
 981	kfree(entry);
 982	return NULL;
 983}
 984
 985#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
 986/* When called from bridge netfilter, skb->data must point to MAC header
 987 * before calling skb_gso_segment(). Else, original MAC header is lost
 988 * and segmented skbs will be sent to wrong destination.
 989 */
 990static void nf_bridge_adjust_skb_data(struct sk_buff *skb)
 991{
 992	if (nf_bridge_info_get(skb))
 993		__skb_push(skb, skb->network_header - skb->mac_header);
 994}
 995
 996static void nf_bridge_adjust_segmented_data(struct sk_buff *skb)
 997{
 998	if (nf_bridge_info_get(skb))
 999		__skb_pull(skb, skb->network_header - skb->mac_header);
1000}
1001#else
1002#define nf_bridge_adjust_skb_data(s) do {} while (0)
1003#define nf_bridge_adjust_segmented_data(s) do {} while (0)
1004#endif
1005
1006static int
1007__nfqnl_enqueue_packet_gso(struct net *net, struct nfqnl_instance *queue,
1008			   struct sk_buff *skb, struct nf_queue_entry *entry)
1009{
1010	int ret = -ENOMEM;
1011	struct nf_queue_entry *entry_seg;
1012
1013	nf_bridge_adjust_segmented_data(skb);
1014
1015	if (skb->next == NULL) { /* last packet, no need to copy entry */
1016		struct sk_buff *gso_skb = entry->skb;
1017		entry->skb = skb;
1018		ret = __nfqnl_enqueue_packet(net, queue, entry);
1019		if (ret)
1020			entry->skb = gso_skb;
1021		return ret;
1022	}
1023
1024	skb_mark_not_on_list(skb);
1025
1026	entry_seg = nf_queue_entry_dup(entry);
1027	if (entry_seg) {
1028		entry_seg->skb = skb;
1029		ret = __nfqnl_enqueue_packet(net, queue, entry_seg);
1030		if (ret)
1031			nf_queue_entry_free(entry_seg);
1032	}
1033	return ret;
1034}
1035
1036static int
1037nfqnl_enqueue_packet(struct nf_queue_entry *entry, unsigned int queuenum)
1038{
1039	struct sk_buff *skb, *segs, *nskb;
1040	bool ct_is_unconfirmed = false;
1041	struct nfqnl_instance *queue;
1042	unsigned int queued;
1043	int err = -ENOBUFS;
1044	struct net *net = entry->state.net;
1045	struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1046
1047	/* rcu_read_lock()ed by nf_hook_thresh */
1048	queue = instance_lookup(q, queuenum);
1049	if (!queue)
1050		return -ESRCH;
1051
1052	if (queue->copy_mode == NFQNL_COPY_NONE)
1053		return -EINVAL;
1054
1055	skb = entry->skb;
1056
1057	switch (entry->state.pf) {
1058	case NFPROTO_IPV4:
1059		skb->protocol = htons(ETH_P_IP);
1060		break;
1061	case NFPROTO_IPV6:
1062		skb->protocol = htons(ETH_P_IPV6);
1063		break;
1064	}
1065
1066	/* Check if someone already holds another reference to
1067	 * unconfirmed ct.  If so, we cannot queue the skb:
1068	 * concurrent modifications of nf_conn->ext are not
1069	 * allowed and we can't know if another CPU isn't
1070	 * processing the same nf_conn entry in parallel.
1071	 */
1072	if (nf_ct_drop_unconfirmed(entry, &ct_is_unconfirmed))
1073		return -EINVAL;
1074
1075	if (!skb_is_gso(skb) || ((queue->flags & NFQA_CFG_F_GSO) && !skb_is_gso_sctp(skb)))
1076		return __nfqnl_enqueue_packet(net, queue, entry);
1077
1078	nf_bridge_adjust_skb_data(skb);
1079	segs = skb_gso_segment(skb, 0);
1080	/* Does not use PTR_ERR to limit the number of error codes that can be
1081	 * returned by nf_queue.  For instance, callers rely on -ESRCH to
1082	 * mean 'ignore this hook'.
1083	 */
1084	if (IS_ERR_OR_NULL(segs))
1085		goto out_err;
1086	queued = 0;
1087	err = 0;
1088
1089	skb_list_walk_safe(segs, segs, nskb) {
1090		if (ct_is_unconfirmed && queued > 0) {
1091			/* skb_gso_segment() increments the ct refcount.
1092			 * This is a problem for unconfirmed (not in hash)
1093			 * entries, those can race when reinjections happen
1094			 * in parallel.
1095			 *
1096			 * Annotate this for all queued entries except the
1097			 * first one.
1098			 *
1099			 * As long as the first one is reinjected first it
1100			 * will do the confirmation for us.
1101			 */
1102			entry->nf_ct_is_unconfirmed = ct_is_unconfirmed;
1103		}
1104
1105		if (err == 0)
1106			err = __nfqnl_enqueue_packet_gso(net, queue,
1107							segs, entry);
1108		if (err == 0)
1109			queued++;
1110		else
1111			kfree_skb(segs);
1112	}
1113
1114	if (queued) {
1115		if (err) /* some segments are already queued */
1116			nf_queue_entry_free(entry);
1117		kfree_skb(skb);
1118		return 0;
1119	}
1120 out_err:
1121	nf_bridge_adjust_segmented_data(skb);
1122	return err;
1123}
1124
1125static int
1126nfqnl_mangle(void *data, unsigned int data_len, struct nf_queue_entry *e, int diff)
1127{
1128	struct sk_buff *nskb;
1129
1130	if (diff < 0) {
1131		unsigned int min_len = skb_transport_offset(e->skb);
1132
1133		if (data_len < min_len)
1134			return -EINVAL;
1135
1136		if (pskb_trim(e->skb, data_len))
1137			return -ENOMEM;
1138	} else if (diff > 0) {
1139		if (data_len > 0xFFFF)
1140			return -EINVAL;
1141		if (diff > skb_tailroom(e->skb)) {
1142			nskb = skb_copy_expand(e->skb, skb_headroom(e->skb),
1143					       diff, GFP_ATOMIC);
1144			if (!nskb)
1145				return -ENOMEM;
1146			kfree_skb(e->skb);
1147			e->skb = nskb;
1148		}
1149		skb_put(e->skb, diff);
1150	}
1151	if (skb_ensure_writable(e->skb, data_len))
1152		return -ENOMEM;
1153	skb_copy_to_linear_data(e->skb, data, data_len);
1154	e->skb->ip_summed = CHECKSUM_NONE;
1155	return 0;
1156}
1157
1158static int
1159nfqnl_set_mode(struct nfqnl_instance *queue,
1160	       unsigned char mode, unsigned int range)
1161{
1162	int status = 0;
1163
1164	spin_lock_bh(&queue->lock);
1165	switch (mode) {
1166	case NFQNL_COPY_NONE:
1167	case NFQNL_COPY_META:
1168		queue->copy_mode = mode;
1169		queue->copy_range = 0;
1170		break;
1171
1172	case NFQNL_COPY_PACKET:
1173		queue->copy_mode = mode;
1174		if (range == 0 || range > NFQNL_MAX_COPY_RANGE)
1175			queue->copy_range = NFQNL_MAX_COPY_RANGE;
1176		else
1177			queue->copy_range = range;
1178		break;
1179
1180	default:
1181		status = -EINVAL;
1182
1183	}
1184	spin_unlock_bh(&queue->lock);
1185
1186	return status;
1187}
1188
1189static int
1190dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex)
1191{
1192#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
1193	int physinif, physoutif;
1194
1195	physinif = nf_bridge_get_physinif(entry->skb);
1196	physoutif = nf_bridge_get_physoutif(entry->skb);
1197
1198	if (physinif == ifindex || physoutif == ifindex)
1199		return 1;
1200#endif
1201	if (entry->state.in)
1202		if (entry->state.in->ifindex == ifindex)
1203			return 1;
1204	if (entry->state.out)
1205		if (entry->state.out->ifindex == ifindex)
1206			return 1;
1207
1208	return 0;
1209}
1210
1211/* drop all packets with either indev or outdev == ifindex from all queue
1212 * instances */
1213static void
1214nfqnl_dev_drop(struct net *net, int ifindex)
1215{
1216	int i;
1217	struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1218
1219	rcu_read_lock();
1220
1221	for (i = 0; i < INSTANCE_BUCKETS; i++) {
1222		struct nfqnl_instance *inst;
1223		struct hlist_head *head = &q->instance_table[i];
1224
1225		hlist_for_each_entry_rcu(inst, head, hlist)
1226			nfqnl_flush(inst, dev_cmp, ifindex);
1227	}
1228
1229	rcu_read_unlock();
1230}
1231
1232static int
1233nfqnl_rcv_dev_event(struct notifier_block *this,
1234		    unsigned long event, void *ptr)
1235{
1236	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1237
1238	/* Drop any packets associated with the downed device */
1239	if (event == NETDEV_DOWN)
1240		nfqnl_dev_drop(dev_net(dev), dev->ifindex);
1241	return NOTIFY_DONE;
1242}
1243
1244static struct notifier_block nfqnl_dev_notifier = {
1245	.notifier_call	= nfqnl_rcv_dev_event,
1246};
1247
1248static void nfqnl_nf_hook_drop(struct net *net)
1249{
1250	struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1251	int i;
1252
1253	/* This function is also called on net namespace error unwind,
1254	 * when pernet_ops->init() failed and ->exit() functions of the
1255	 * previous pernet_ops gets called.
1256	 *
1257	 * This may result in a call to nfqnl_nf_hook_drop() before
1258	 * struct nfnl_queue_net was allocated.
1259	 */
1260	if (!q)
1261		return;
1262
1263	for (i = 0; i < INSTANCE_BUCKETS; i++) {
1264		struct nfqnl_instance *inst;
1265		struct hlist_head *head = &q->instance_table[i];
1266
1267		hlist_for_each_entry_rcu(inst, head, hlist)
1268			nfqnl_flush(inst, NULL, 0);
1269	}
1270}
1271
1272static int
1273nfqnl_rcv_nl_event(struct notifier_block *this,
1274		   unsigned long event, void *ptr)
1275{
1276	struct netlink_notify *n = ptr;
1277	struct nfnl_queue_net *q = nfnl_queue_pernet(n->net);
1278
1279	if (event == NETLINK_URELEASE && n->protocol == NETLINK_NETFILTER) {
1280		int i;
1281
1282		/* destroy all instances for this portid */
1283		spin_lock(&q->instances_lock);
1284		for (i = 0; i < INSTANCE_BUCKETS; i++) {
1285			struct hlist_node *t2;
1286			struct nfqnl_instance *inst;
1287			struct hlist_head *head = &q->instance_table[i];
1288
1289			hlist_for_each_entry_safe(inst, t2, head, hlist) {
1290				if (n->portid == inst->peer_portid)
1291					__instance_destroy(inst);
1292			}
1293		}
1294		spin_unlock(&q->instances_lock);
1295	}
1296	return NOTIFY_DONE;
1297}
1298
1299static struct notifier_block nfqnl_rtnl_notifier = {
1300	.notifier_call	= nfqnl_rcv_nl_event,
1301};
1302
1303static const struct nla_policy nfqa_vlan_policy[NFQA_VLAN_MAX + 1] = {
1304	[NFQA_VLAN_TCI]		= { .type = NLA_U16},
1305	[NFQA_VLAN_PROTO]	= { .type = NLA_U16},
1306};
1307
1308static const struct nla_policy nfqa_verdict_policy[NFQA_MAX+1] = {
1309	[NFQA_VERDICT_HDR]	= { .len = sizeof(struct nfqnl_msg_verdict_hdr) },
1310	[NFQA_MARK]		= { .type = NLA_U32 },
1311	[NFQA_PAYLOAD]		= { .type = NLA_UNSPEC },
1312	[NFQA_CT]		= { .type = NLA_UNSPEC },
1313	[NFQA_EXP]		= { .type = NLA_UNSPEC },
1314	[NFQA_VLAN]		= { .type = NLA_NESTED },
1315	[NFQA_PRIORITY]		= { .type = NLA_U32 },
1316};
1317
1318static const struct nla_policy nfqa_verdict_batch_policy[NFQA_MAX+1] = {
1319	[NFQA_VERDICT_HDR]	= { .len = sizeof(struct nfqnl_msg_verdict_hdr) },
1320	[NFQA_MARK]		= { .type = NLA_U32 },
1321	[NFQA_PRIORITY]		= { .type = NLA_U32 },
1322};
1323
1324static struct nfqnl_instance *
1325verdict_instance_lookup(struct nfnl_queue_net *q, u16 queue_num, u32 nlportid)
1326{
1327	struct nfqnl_instance *queue;
1328
1329	queue = instance_lookup(q, queue_num);
1330	if (!queue)
1331		return ERR_PTR(-ENODEV);
1332
1333	if (queue->peer_portid != nlportid)
1334		return ERR_PTR(-EPERM);
1335
1336	return queue;
1337}
1338
1339static struct nfqnl_msg_verdict_hdr*
1340verdicthdr_get(const struct nlattr * const nfqa[])
1341{
1342	struct nfqnl_msg_verdict_hdr *vhdr;
1343	unsigned int verdict;
1344
1345	if (!nfqa[NFQA_VERDICT_HDR])
1346		return NULL;
1347
1348	vhdr = nla_data(nfqa[NFQA_VERDICT_HDR]);
1349	verdict = ntohl(vhdr->verdict) & NF_VERDICT_MASK;
1350	if (verdict > NF_MAX_VERDICT || verdict == NF_STOLEN)
1351		return NULL;
1352	return vhdr;
1353}
1354
1355static int nfq_id_after(unsigned int id, unsigned int max)
1356{
1357	return (int)(id - max) > 0;
1358}
1359
1360static int nfqnl_recv_verdict_batch(struct sk_buff *skb,
1361				    const struct nfnl_info *info,
1362				    const struct nlattr * const nfqa[])
1363{
1364	struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1365	u16 queue_num = ntohs(info->nfmsg->res_id);
1366	struct nf_queue_entry *entry, *tmp;
1367	struct nfqnl_msg_verdict_hdr *vhdr;
1368	struct nfqnl_instance *queue;
1369	unsigned int verdict, maxid;
1370	LIST_HEAD(batch_list);
1371
1372	queue = verdict_instance_lookup(q, queue_num,
1373					NETLINK_CB(skb).portid);
1374	if (IS_ERR(queue))
1375		return PTR_ERR(queue);
1376
1377	vhdr = verdicthdr_get(nfqa);
1378	if (!vhdr)
1379		return -EINVAL;
1380
1381	verdict = ntohl(vhdr->verdict);
1382	maxid = ntohl(vhdr->id);
1383
1384	spin_lock_bh(&queue->lock);
1385
1386	list_for_each_entry_safe(entry, tmp, &queue->queue_list, list) {
1387		if (nfq_id_after(entry->id, maxid))
1388			break;
1389		__dequeue_entry(queue, entry);
1390		list_add_tail(&entry->list, &batch_list);
1391	}
1392
1393	spin_unlock_bh(&queue->lock);
1394
1395	if (list_empty(&batch_list))
1396		return -ENOENT;
1397
1398	list_for_each_entry_safe(entry, tmp, &batch_list, list) {
1399		if (nfqa[NFQA_MARK])
1400			entry->skb->mark = ntohl(nla_get_be32(nfqa[NFQA_MARK]));
1401
1402		if (nfqa[NFQA_PRIORITY])
1403			entry->skb->priority = ntohl(nla_get_be32(nfqa[NFQA_PRIORITY]));
1404
1405		nfqnl_reinject(entry, verdict);
1406	}
1407	return 0;
1408}
1409
1410static struct nf_conn *nfqnl_ct_parse(const struct nfnl_ct_hook *nfnl_ct,
1411				      const struct nlmsghdr *nlh,
1412				      const struct nlattr * const nfqa[],
1413				      struct nf_queue_entry *entry,
1414				      enum ip_conntrack_info *ctinfo)
1415{
1416#if IS_ENABLED(CONFIG_NF_CONNTRACK)
1417	struct nf_conn *ct;
1418
1419	ct = nf_ct_get(entry->skb, ctinfo);
1420	if (ct == NULL)
1421		return NULL;
1422
1423	if (nfnl_ct->parse(nfqa[NFQA_CT], ct) < 0)
1424		return NULL;
1425
1426	if (nfqa[NFQA_EXP])
1427		nfnl_ct->attach_expect(nfqa[NFQA_EXP], ct,
1428				      NETLINK_CB(entry->skb).portid,
1429				      nlmsg_report(nlh));
1430	return ct;
1431#else
1432	return NULL;
1433#endif
1434}
1435
1436static int nfqa_parse_bridge(struct nf_queue_entry *entry,
1437			     const struct nlattr * const nfqa[])
1438{
1439	if (nfqa[NFQA_VLAN]) {
1440		struct nlattr *tb[NFQA_VLAN_MAX + 1];
1441		int err;
1442
1443		err = nla_parse_nested_deprecated(tb, NFQA_VLAN_MAX,
1444						  nfqa[NFQA_VLAN],
1445						  nfqa_vlan_policy, NULL);
1446		if (err < 0)
1447			return err;
1448
1449		if (!tb[NFQA_VLAN_TCI] || !tb[NFQA_VLAN_PROTO])
1450			return -EINVAL;
1451
1452		__vlan_hwaccel_put_tag(entry->skb,
1453			nla_get_be16(tb[NFQA_VLAN_PROTO]),
1454			ntohs(nla_get_be16(tb[NFQA_VLAN_TCI])));
1455	}
1456
1457	if (nfqa[NFQA_L2HDR]) {
1458		int mac_header_len = entry->skb->network_header -
1459			entry->skb->mac_header;
1460
1461		if (mac_header_len != nla_len(nfqa[NFQA_L2HDR]))
1462			return -EINVAL;
1463		else if (mac_header_len > 0)
1464			memcpy(skb_mac_header(entry->skb),
1465			       nla_data(nfqa[NFQA_L2HDR]),
1466			       mac_header_len);
1467	}
1468
1469	return 0;
1470}
1471
1472static int nfqnl_recv_verdict(struct sk_buff *skb, const struct nfnl_info *info,
1473			      const struct nlattr * const nfqa[])
1474{
1475	struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1476	u_int16_t queue_num = ntohs(info->nfmsg->res_id);
1477	const struct nfnl_ct_hook *nfnl_ct;
1478	struct nfqnl_msg_verdict_hdr *vhdr;
1479	enum ip_conntrack_info ctinfo;
1480	struct nfqnl_instance *queue;
1481	struct nf_queue_entry *entry;
1482	struct nf_conn *ct = NULL;
1483	unsigned int verdict;
1484	int err;
1485
1486	queue = verdict_instance_lookup(q, queue_num,
1487					NETLINK_CB(skb).portid);
1488	if (IS_ERR(queue))
1489		return PTR_ERR(queue);
1490
1491	vhdr = verdicthdr_get(nfqa);
1492	if (!vhdr)
1493		return -EINVAL;
1494
1495	verdict = ntohl(vhdr->verdict);
1496
1497	entry = find_dequeue_entry(queue, ntohl(vhdr->id));
1498	if (entry == NULL)
1499		return -ENOENT;
1500
1501	/* rcu lock already held from nfnl->call_rcu. */
1502	nfnl_ct = rcu_dereference(nfnl_ct_hook);
1503
1504	if (nfqa[NFQA_CT]) {
1505		if (nfnl_ct != NULL)
1506			ct = nfqnl_ct_parse(nfnl_ct, info->nlh, nfqa, entry,
1507					    &ctinfo);
1508	}
1509
1510	if (entry->state.pf == PF_BRIDGE) {
1511		err = nfqa_parse_bridge(entry, nfqa);
1512		if (err < 0) {
1513			nfqnl_reinject(entry, NF_DROP);
1514			return err;
1515		}
1516	}
1517
1518	if (nfqa[NFQA_PAYLOAD]) {
1519		u16 payload_len = nla_len(nfqa[NFQA_PAYLOAD]);
1520		int diff = payload_len - entry->skb->len;
1521
1522		if (nfqnl_mangle(nla_data(nfqa[NFQA_PAYLOAD]),
1523				 payload_len, entry, diff) < 0)
1524			verdict = NF_DROP;
1525
1526		if (ct && diff)
1527			nfnl_ct->seq_adjust(entry->skb, ct, ctinfo, diff);
1528	}
1529
1530	if (nfqa[NFQA_MARK])
1531		entry->skb->mark = ntohl(nla_get_be32(nfqa[NFQA_MARK]));
1532
1533	if (nfqa[NFQA_PRIORITY])
1534		entry->skb->priority = ntohl(nla_get_be32(nfqa[NFQA_PRIORITY]));
1535
1536	nfqnl_reinject(entry, verdict);
1537	return 0;
1538}
1539
1540static int nfqnl_recv_unsupp(struct sk_buff *skb, const struct nfnl_info *info,
1541			     const struct nlattr * const cda[])
1542{
1543	return -ENOTSUPP;
1544}
1545
1546static const struct nla_policy nfqa_cfg_policy[NFQA_CFG_MAX+1] = {
1547	[NFQA_CFG_CMD]		= { .len = sizeof(struct nfqnl_msg_config_cmd) },
1548	[NFQA_CFG_PARAMS]	= { .len = sizeof(struct nfqnl_msg_config_params) },
1549	[NFQA_CFG_QUEUE_MAXLEN]	= { .type = NLA_U32 },
1550	[NFQA_CFG_MASK]		= { .type = NLA_U32 },
1551	[NFQA_CFG_FLAGS]	= { .type = NLA_U32 },
1552};
1553
1554static const struct nf_queue_handler nfqh = {
1555	.outfn		= nfqnl_enqueue_packet,
1556	.nf_hook_drop	= nfqnl_nf_hook_drop,
1557};
1558
1559static int nfqnl_recv_config(struct sk_buff *skb, const struct nfnl_info *info,
1560			     const struct nlattr * const nfqa[])
1561{
1562	struct nfnl_queue_net *q = nfnl_queue_pernet(info->net);
1563	u_int16_t queue_num = ntohs(info->nfmsg->res_id);
1564	struct nfqnl_msg_config_cmd *cmd = NULL;
1565	struct nfqnl_instance *queue;
1566	__u32 flags = 0, mask = 0;
1567
1568	WARN_ON_ONCE(!lockdep_nfnl_is_held(NFNL_SUBSYS_QUEUE));
1569
1570	if (nfqa[NFQA_CFG_CMD]) {
1571		cmd = nla_data(nfqa[NFQA_CFG_CMD]);
1572
1573		/* Obsolete commands without queue context */
1574		switch (cmd->command) {
1575		case NFQNL_CFG_CMD_PF_BIND: return 0;
1576		case NFQNL_CFG_CMD_PF_UNBIND: return 0;
1577		}
1578	}
1579
1580	/* Check if we support these flags in first place, dependencies should
1581	 * be there too not to break atomicity.
1582	 */
1583	if (nfqa[NFQA_CFG_FLAGS]) {
1584		if (!nfqa[NFQA_CFG_MASK]) {
1585			/* A mask is needed to specify which flags are being
1586			 * changed.
1587			 */
1588			return -EINVAL;
1589		}
1590
1591		flags = ntohl(nla_get_be32(nfqa[NFQA_CFG_FLAGS]));
1592		mask = ntohl(nla_get_be32(nfqa[NFQA_CFG_MASK]));
1593
1594		if (flags >= NFQA_CFG_F_MAX)
1595			return -EOPNOTSUPP;
1596
1597#if !IS_ENABLED(CONFIG_NETWORK_SECMARK)
1598		if (flags & mask & NFQA_CFG_F_SECCTX)
1599			return -EOPNOTSUPP;
1600#endif
1601		if ((flags & mask & NFQA_CFG_F_CONNTRACK) &&
1602		    !rcu_access_pointer(nfnl_ct_hook)) {
1603#ifdef CONFIG_MODULES
1604			nfnl_unlock(NFNL_SUBSYS_QUEUE);
1605			request_module("ip_conntrack_netlink");
1606			nfnl_lock(NFNL_SUBSYS_QUEUE);
1607			if (rcu_access_pointer(nfnl_ct_hook))
1608				return -EAGAIN;
1609#endif
1610			return -EOPNOTSUPP;
1611		}
1612	}
1613
1614	/* Lookup queue under RCU. After peer_portid check (or for new queue
1615	 * in BIND case), the queue is owned by the socket sending this message.
1616	 * A socket cannot simultaneously send a message and close, so while
1617	 * processing this CONFIG message, nfqnl_rcv_nl_event() (triggered by
1618	 * socket close) cannot destroy this queue. Safe to use without RCU.
1619	 */
1620	rcu_read_lock();
1621	queue = instance_lookup(q, queue_num);
1622	if (queue && queue->peer_portid != NETLINK_CB(skb).portid) {
1623		rcu_read_unlock();
1624		return -EPERM;
1625	}
1626	rcu_read_unlock();
1627
1628	if (cmd != NULL) {
1629		switch (cmd->command) {
1630		case NFQNL_CFG_CMD_BIND:
1631			if (queue)
1632				return -EBUSY;
1633			queue = instance_create(q, queue_num, NETLINK_CB(skb).portid);
1634			if (IS_ERR(queue))
1635				return PTR_ERR(queue);
1636			break;
1637		case NFQNL_CFG_CMD_UNBIND:
1638			if (!queue)
1639				return -ENODEV;
1640			instance_destroy(q, queue);
1641			return 0;
1642		case NFQNL_CFG_CMD_PF_BIND:
1643		case NFQNL_CFG_CMD_PF_UNBIND:
1644			break;
1645		default:
1646			return -EOPNOTSUPP;
1647		}
1648	}
1649
1650	if (!queue)
1651		return -ENODEV;
1652
1653	if (nfqa[NFQA_CFG_PARAMS]) {
1654		struct nfqnl_msg_config_params *params =
1655			nla_data(nfqa[NFQA_CFG_PARAMS]);
1656
1657		nfqnl_set_mode(queue, params->copy_mode,
1658				ntohl(params->copy_range));
1659	}
1660
1661	if (nfqa[NFQA_CFG_QUEUE_MAXLEN]) {
1662		__be32 *queue_maxlen = nla_data(nfqa[NFQA_CFG_QUEUE_MAXLEN]);
1663
1664		spin_lock_bh(&queue->lock);
1665		queue->queue_maxlen = ntohl(*queue_maxlen);
1666		spin_unlock_bh(&queue->lock);
1667	}
1668
1669	if (nfqa[NFQA_CFG_FLAGS]) {
1670		spin_lock_bh(&queue->lock);
1671		queue->flags &= ~mask;
1672		queue->flags |= flags & mask;
1673		spin_unlock_bh(&queue->lock);
1674	}
1675
1676	return 0;
1677}
1678
1679static const struct nfnl_callback nfqnl_cb[NFQNL_MSG_MAX] = {
1680	[NFQNL_MSG_PACKET]	= {
1681		.call		= nfqnl_recv_unsupp,
1682		.type		= NFNL_CB_RCU,
1683		.attr_count	= NFQA_MAX,
1684	},
1685	[NFQNL_MSG_VERDICT]	= {
1686		.call		= nfqnl_recv_verdict,
1687		.type		= NFNL_CB_RCU,
1688		.attr_count	= NFQA_MAX,
1689		.policy		= nfqa_verdict_policy
1690	},
1691	[NFQNL_MSG_CONFIG]	= {
1692		.call		= nfqnl_recv_config,
1693		.type		= NFNL_CB_MUTEX,
1694		.attr_count	= NFQA_CFG_MAX,
1695		.policy		= nfqa_cfg_policy
1696	},
1697	[NFQNL_MSG_VERDICT_BATCH] = {
1698		.call		= nfqnl_recv_verdict_batch,
1699		.type		= NFNL_CB_RCU,
1700		.attr_count	= NFQA_MAX,
1701		.policy		= nfqa_verdict_batch_policy
1702	},
1703};
1704
1705static const struct nfnetlink_subsystem nfqnl_subsys = {
1706	.name		= "nf_queue",
1707	.subsys_id	= NFNL_SUBSYS_QUEUE,
1708	.cb_count	= NFQNL_MSG_MAX,
1709	.cb		= nfqnl_cb,
1710};
1711
1712#ifdef CONFIG_PROC_FS
1713struct iter_state {
1714	struct seq_net_private p;
1715	unsigned int bucket;
1716};
1717
1718static struct hlist_node *get_first(struct seq_file *seq)
1719{
1720	struct iter_state *st = seq->private;
1721	struct net *net;
1722	struct nfnl_queue_net *q;
1723
1724	if (!st)
1725		return NULL;
1726
1727	net = seq_file_net(seq);
1728	q = nfnl_queue_pernet(net);
1729	for (st->bucket = 0; st->bucket < INSTANCE_BUCKETS; st->bucket++) {
1730		if (!hlist_empty(&q->instance_table[st->bucket]))
1731			return q->instance_table[st->bucket].first;
1732	}
1733	return NULL;
1734}
1735
1736static struct hlist_node *get_next(struct seq_file *seq, struct hlist_node *h)
1737{
1738	struct iter_state *st = seq->private;
1739	struct net *net = seq_file_net(seq);
1740
1741	h = h->next;
1742	while (!h) {
1743		struct nfnl_queue_net *q;
1744
1745		if (++st->bucket >= INSTANCE_BUCKETS)
1746			return NULL;
1747
1748		q = nfnl_queue_pernet(net);
1749		h = q->instance_table[st->bucket].first;
1750	}
1751	return h;
1752}
1753
1754static struct hlist_node *get_idx(struct seq_file *seq, loff_t pos)
1755{
1756	struct hlist_node *head;
1757	head = get_first(seq);
1758
1759	if (head)
1760		while (pos && (head = get_next(seq, head)))
1761			pos--;
1762	return pos ? NULL : head;
1763}
1764
1765static void *seq_start(struct seq_file *s, loff_t *pos)
1766	__acquires(nfnl_queue_pernet(seq_file_net(s))->instances_lock)
1767{
1768	spin_lock(&nfnl_queue_pernet(seq_file_net(s))->instances_lock);
1769	return get_idx(s, *pos);
1770}
1771
1772static void *seq_next(struct seq_file *s, void *v, loff_t *pos)
1773{
1774	(*pos)++;
1775	return get_next(s, v);
1776}
1777
1778static void seq_stop(struct seq_file *s, void *v)
1779	__releases(nfnl_queue_pernet(seq_file_net(s))->instances_lock)
1780{
1781	spin_unlock(&nfnl_queue_pernet(seq_file_net(s))->instances_lock);
1782}
1783
1784static int seq_show(struct seq_file *s, void *v)
1785{
1786	const struct nfqnl_instance *inst = v;
1787
1788	seq_printf(s, "%5u %6u %5u %1u %5u %5u %5u %8u %2d\n",
1789		   inst->queue_num,
1790		   inst->peer_portid, inst->queue_total,
1791		   inst->copy_mode, inst->copy_range,
1792		   inst->queue_dropped, inst->queue_user_dropped,
1793		   inst->id_sequence, 1);
1794	return 0;
1795}
1796
1797static const struct seq_operations nfqnl_seq_ops = {
1798	.start	= seq_start,
1799	.next	= seq_next,
1800	.stop	= seq_stop,
1801	.show	= seq_show,
1802};
1803#endif /* PROC_FS */
1804
1805static int __net_init nfnl_queue_net_init(struct net *net)
1806{
1807	unsigned int i;
1808	struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1809
1810	for (i = 0; i < INSTANCE_BUCKETS; i++)
1811		INIT_HLIST_HEAD(&q->instance_table[i]);
1812
1813	spin_lock_init(&q->instances_lock);
1814
1815#ifdef CONFIG_PROC_FS
1816	if (!proc_create_net("nfnetlink_queue", 0440, net->nf.proc_netfilter,
1817			&nfqnl_seq_ops, sizeof(struct iter_state)))
1818		return -ENOMEM;
1819#endif
1820	return 0;
1821}
1822
1823static void __net_exit nfnl_queue_net_exit(struct net *net)
1824{
1825	struct nfnl_queue_net *q = nfnl_queue_pernet(net);
1826	unsigned int i;
1827
1828#ifdef CONFIG_PROC_FS
1829	remove_proc_entry("nfnetlink_queue", net->nf.proc_netfilter);
1830#endif
1831	for (i = 0; i < INSTANCE_BUCKETS; i++)
1832		WARN_ON_ONCE(!hlist_empty(&q->instance_table[i]));
1833}
1834
1835static struct pernet_operations nfnl_queue_net_ops = {
1836	.init		= nfnl_queue_net_init,
1837	.exit		= nfnl_queue_net_exit,
1838	.id		= &nfnl_queue_net_id,
1839	.size		= sizeof(struct nfnl_queue_net),
1840};
1841
1842static int __init nfnetlink_queue_init(void)
1843{
1844	int status;
1845
1846	nfq_cleanup_wq = alloc_ordered_workqueue("nfq_workqueue", 0);
1847	if (!nfq_cleanup_wq)
1848		return -ENOMEM;
1849
1850	status = register_pernet_subsys(&nfnl_queue_net_ops);
1851	if (status < 0)
1852		goto cleanup_pernet_subsys;
1853
1854	status = netlink_register_notifier(&nfqnl_rtnl_notifier);
1855	if (status < 0)
1856	       goto cleanup_rtnl_notifier;
1857
1858	status = register_netdevice_notifier(&nfqnl_dev_notifier);
1859	if (status < 0)
1860		goto cleanup_dev_notifier;
1861
1862	status = nfnetlink_subsys_register(&nfqnl_subsys);
1863	if (status < 0)
1864		goto cleanup_nfqnl_subsys;
1865
1866	nf_register_queue_handler(&nfqh);
1867
1868	return status;
1869
1870cleanup_nfqnl_subsys:
1871	unregister_netdevice_notifier(&nfqnl_dev_notifier);
1872cleanup_dev_notifier:
1873	netlink_unregister_notifier(&nfqnl_rtnl_notifier);
1874cleanup_rtnl_notifier:
1875	unregister_pernet_subsys(&nfnl_queue_net_ops);
1876cleanup_pernet_subsys:
1877	destroy_workqueue(nfq_cleanup_wq);
1878	return status;
1879}
1880
1881static void __exit nfnetlink_queue_fini(void)
1882{
1883	nf_unregister_queue_handler();
1884	unregister_netdevice_notifier(&nfqnl_dev_notifier);
1885	nfnetlink_subsys_unregister(&nfqnl_subsys);
1886	netlink_unregister_notifier(&nfqnl_rtnl_notifier);
1887	unregister_pernet_subsys(&nfnl_queue_net_ops);
1888	destroy_workqueue(nfq_cleanup_wq);
1889	rcu_barrier(); /* Wait for completion of call_rcu()'s */
1890}
1891
1892MODULE_DESCRIPTION("netfilter packet queue handler");
1893MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>");
1894MODULE_LICENSE("GPL");
1895MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_QUEUE);
1896
1897module_init(nfnetlink_queue_init);
1898module_exit(nfnetlink_queue_fini);