udp.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
   4 *		operating system.  INET is implemented using the  BSD Socket
   5 *		interface as the means of communication with the user level.
   6 *
   7 *		The User Datagram Protocol (UDP).
   8 *
   9 * Authors:	Ross Biro
  10 *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  12 *		Alan Cox, <alan@lxorguk.ukuu.org.uk>
  13 *		Hirokazu Takahashi, <taka@valinux.co.jp>
  14 *
  15 * Fixes:
  16 *		Alan Cox	:	verify_area() calls
  17 *		Alan Cox	: 	stopped close while in use off icmp
  18 *					messages. Not a fix but a botch that
  19 *					for udp at least is 'valid'.
  20 *		Alan Cox	:	Fixed icmp handling properly
  21 *		Alan Cox	: 	Correct error for oversized datagrams
  22 *		Alan Cox	:	Tidied select() semantics.
  23 *		Alan Cox	:	udp_err() fixed properly, also now
  24 *					select and read wake correctly on errors
  25 *		Alan Cox	:	udp_send verify_area moved to avoid mem leak
  26 *		Alan Cox	:	UDP can count its memory
  27 *		Alan Cox	:	send to an unknown connection causes
  28 *					an ECONNREFUSED off the icmp, but
  29 *					does NOT close.
  30 *		Alan Cox	:	Switched to new sk_buff handlers. No more backlog!
  31 *		Alan Cox	:	Using generic datagram code. Even smaller and the PEEK
  32 *					bug no longer crashes it.
  33 *		Fred Van Kempen	: 	Net2e support for sk->broadcast.
  34 *		Alan Cox	:	Uses skb_free_datagram
  35 *		Alan Cox	:	Added get/set sockopt support.
  36 *		Alan Cox	:	Broadcasting without option set returns EACCES.
  37 *		Alan Cox	:	No wakeup calls. Instead we now use the callbacks.
  38 *		Alan Cox	:	Use ip_tos and ip_ttl
  39 *		Alan Cox	:	SNMP Mibs
  40 *		Alan Cox	:	MSG_DONTROUTE, and 0.0.0.0 support.
  41 *		Matt Dillon	:	UDP length checks.
  42 *		Alan Cox	:	Smarter af_inet used properly.
  43 *		Alan Cox	:	Use new kernel side addressing.
  44 *		Alan Cox	:	Incorrect return on truncated datagram receive.
  45 *	Arnt Gulbrandsen 	:	New udp_send and stuff
  46 *		Alan Cox	:	Cache last socket
  47 *		Alan Cox	:	Route cache
  48 *		Jon Peatfield	:	Minor efficiency fix to sendto().
  49 *		Mike Shaver	:	RFC1122 checks.
  50 *		Alan Cox	:	Nonblocking error fix.
  51 *	Willy Konynenberg	:	Transparent proxying support.
  52 *		Mike McLagan	:	Routing by source
  53 *		David S. Miller	:	New socket lookup architecture.
  54 *					Last socket cache retained as it
  55 *					does have a high hit rate.
  56 *		Olaf Kirch	:	Don't linearise iovec on sendmsg.
  57 *		Andi Kleen	:	Some cleanups, cache destination entry
  58 *					for connect.
  59 *	Vitaly E. Lavrov	:	Transparent proxy revived after year coma.
  60 *		Melvin Smith	:	Check msg_name not msg_namelen in sendto(),
  61 *					return ENOTCONN for unconnected sockets (POSIX)
  62 *		Janos Farkas	:	don't deliver multi/broadcasts to a different
  63 *					bound-to-device socket
  64 *	Hirokazu Takahashi	:	HW checksumming for outgoing UDP
  65 *					datagrams.
  66 *	Hirokazu Takahashi	:	sendfile() on UDP works now.
  67 *		Arnaldo C. Melo :	convert /proc/net/udp to seq_file
  68 *	YOSHIFUJI Hideaki @USAGI and:	Support IPV6_V6ONLY socket option, which
  69 *	Alexey Kuznetsov:		allow both IPv4 and IPv6 sockets to bind
  70 *					a single port at the same time.
  71 *	Derek Atkins <derek@ihtfp.com>: Add Encapsulation Support
  72 *	James Chapman		:	Add L2TP encapsulation type.
  73 */
  74
  75#define pr_fmt(fmt) "UDP: " fmt
  76
  77#include <linux/bpf-cgroup.h>
  78#include <linux/uaccess.h>
  79#include <asm/ioctls.h>
  80#include <linux/memblock.h>
  81#include <linux/highmem.h>
  82#include <linux/types.h>
  83#include <linux/fcntl.h>
  84#include <linux/module.h>
  85#include <linux/socket.h>
  86#include <linux/sockios.h>
  87#include <linux/igmp.h>
  88#include <linux/inetdevice.h>
  89#include <linux/in.h>
  90#include <linux/errno.h>
  91#include <linux/timer.h>
  92#include <linux/mm.h>
  93#include <linux/inet.h>
  94#include <linux/netdevice.h>
  95#include <linux/slab.h>
  96#include <linux/sock_diag.h>
  97#include <net/tcp_states.h>
  98#include <linux/skbuff.h>
  99#include <linux/proc_fs.h>
 100#include <linux/seq_file.h>
 101#include <net/net_namespace.h>
 102#include <net/icmp.h>
 103#include <net/inet_hashtables.h>
 104#include <net/ip.h>
 105#include <net/ip_tunnels.h>
 106#include <net/route.h>
 107#include <net/checksum.h>
 108#include <net/gso.h>
 109#include <net/xfrm.h>
 110#include <trace/events/udp.h>
 111#include <linux/static_key.h>
 112#include <linux/btf_ids.h>
 113#include <trace/events/skb.h>
 114#include <net/busy_poll.h>
 115#include "udp_impl.h"
 116#include <net/sock_reuseport.h>
 117#include <net/addrconf.h>
 118#include <net/udp_tunnel.h>
 119#include <net/gro.h>
 120#if IS_ENABLED(CONFIG_IPV6)
 121#include <net/ipv6_stubs.h>
 122#endif
 123#include <net/rps.h>
 124
 125struct udp_table udp_table __read_mostly;
 126
 127long sysctl_udp_mem[3] __read_mostly;
 128EXPORT_IPV6_MOD(sysctl_udp_mem);
 129
 130DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
 131EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc);
 132
 133#define MAX_UDP_PORTS 65536
 134#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN_PERNET)
 135
 136static struct udp_table *udp_get_table_prot(struct sock *sk)
 137{
 138	return sk->sk_prot->h.udp_table ? : sock_net(sk)->ipv4.udp_table;
 139}
 140
 141static int udp_lib_lport_inuse(struct net *net, __u16 num,
 142			       const struct udp_hslot *hslot,
 143			       unsigned long *bitmap,
 144			       struct sock *sk, unsigned int log)
 145{
 146	kuid_t uid = sk_uid(sk);
 147	struct sock *sk2;
 148
 149	sk_for_each(sk2, &hslot->head) {
 150		if (net_eq(sock_net(sk2), net) &&
 151		    sk2 != sk &&
 152		    (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
 153		    (!sk2->sk_reuse || !sk->sk_reuse) &&
 154		    (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
 155		     sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
 156		    inet_rcv_saddr_equal(sk, sk2, true)) {
 157			if (sk2->sk_reuseport && sk->sk_reuseport &&
 158			    !rcu_access_pointer(sk->sk_reuseport_cb) &&
 159			    uid_eq(uid, sk_uid(sk2))) {
 160				if (!bitmap)
 161					return 0;
 162			} else {
 163				if (!bitmap)
 164					return 1;
 165				__set_bit(udp_sk(sk2)->udp_port_hash >> log,
 166					  bitmap);
 167			}
 168		}
 169	}
 170	return 0;
 171}
 172
 173/*
 174 * Note: we still hold spinlock of primary hash chain, so no other writer
 175 * can insert/delete a socket with local_port == num
 176 */
 177static int udp_lib_lport_inuse2(struct net *net, __u16 num,
 178				struct udp_hslot *hslot2,
 179				struct sock *sk)
 180{
 181	kuid_t uid = sk_uid(sk);
 182	struct sock *sk2;
 183	int res = 0;
 184
 185	spin_lock(&hslot2->lock);
 186	udp_portaddr_for_each_entry(sk2, &hslot2->head) {
 187		if (net_eq(sock_net(sk2), net) &&
 188		    sk2 != sk &&
 189		    (udp_sk(sk2)->udp_port_hash == num) &&
 190		    (!sk2->sk_reuse || !sk->sk_reuse) &&
 191		    (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
 192		     sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
 193		    inet_rcv_saddr_equal(sk, sk2, true)) {
 194			if (sk2->sk_reuseport && sk->sk_reuseport &&
 195			    !rcu_access_pointer(sk->sk_reuseport_cb) &&
 196			    uid_eq(uid, sk_uid(sk2))) {
 197				res = 0;
 198			} else {
 199				res = 1;
 200			}
 201			break;
 202		}
 203	}
 204	spin_unlock(&hslot2->lock);
 205	return res;
 206}
 207
 208static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
 209{
 210	struct net *net = sock_net(sk);
 211	kuid_t uid = sk_uid(sk);
 212	struct sock *sk2;
 213
 214	sk_for_each(sk2, &hslot->head) {
 215		if (net_eq(sock_net(sk2), net) &&
 216		    sk2 != sk &&
 217		    sk2->sk_family == sk->sk_family &&
 218		    ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
 219		    (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
 220		    (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
 221		    sk2->sk_reuseport && uid_eq(uid, sk_uid(sk2)) &&
 222		    inet_rcv_saddr_equal(sk, sk2, false)) {
 223			return reuseport_add_sock(sk, sk2,
 224						  inet_rcv_saddr_any(sk));
 225		}
 226	}
 227
 228	return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
 229}
 230
 231/**
 232 *  udp_lib_get_port  -  UDP/-Lite port lookup for IPv4 and IPv6
 233 *
 234 *  @sk:          socket struct in question
 235 *  @snum:        port number to look up
 236 *  @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
 237 *                   with NULL address
 238 */
 239int udp_lib_get_port(struct sock *sk, unsigned short snum,
 240		     unsigned int hash2_nulladdr)
 241{
 242	struct udp_table *udptable = udp_get_table_prot(sk);
 243	struct udp_hslot *hslot, *hslot2;
 244	struct net *net = sock_net(sk);
 245	int error = -EADDRINUSE;
 246
 247	if (!snum) {
 248		DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
 249		unsigned short first, last;
 250		int low, high, remaining;
 251		unsigned int rand;
 252
 253		inet_sk_get_local_port_range(sk, &low, &high);
 254		remaining = (high - low) + 1;
 255
 256		rand = get_random_u32();
 257		first = reciprocal_scale(rand, remaining) + low;
 258		/*
 259		 * force rand to be an odd multiple of UDP_HTABLE_SIZE
 260		 */
 261		rand = (rand | 1) * (udptable->mask + 1);
 262		last = first + udptable->mask + 1;
 263		do {
 264			hslot = udp_hashslot(udptable, net, first);
 265			bitmap_zero(bitmap, PORTS_PER_CHAIN);
 266			spin_lock_bh(&hslot->lock);
 267			udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
 268					    udptable->log);
 269
 270			snum = first;
 271			/*
 272			 * Iterate on all possible values of snum for this hash.
 273			 * Using steps of an odd multiple of UDP_HTABLE_SIZE
 274			 * give us randomization and full range coverage.
 275			 */
 276			do {
 277				if (low <= snum && snum <= high &&
 278				    !test_bit(snum >> udptable->log, bitmap) &&
 279				    !inet_is_local_reserved_port(net, snum))
 280					goto found;
 281				snum += rand;
 282			} while (snum != first);
 283			spin_unlock_bh(&hslot->lock);
 284			cond_resched();
 285		} while (++first != last);
 286		goto fail;
 287	} else {
 288		hslot = udp_hashslot(udptable, net, snum);
 289		spin_lock_bh(&hslot->lock);
 290		if (inet_use_hash2_on_bind(sk) && hslot->count > 10) {
 291			int exist;
 292			unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
 293
 294			slot2          &= udptable->mask;
 295			hash2_nulladdr &= udptable->mask;
 296
 297			hslot2 = udp_hashslot2(udptable, slot2);
 298			if (hslot->count < hslot2->count)
 299				goto scan_primary_hash;
 300
 301			exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
 302			if (!exist && (hash2_nulladdr != slot2)) {
 303				hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
 304				exist = udp_lib_lport_inuse2(net, snum, hslot2,
 305							     sk);
 306			}
 307			if (exist)
 308				goto fail_unlock;
 309			else
 310				goto found;
 311		}
 312scan_primary_hash:
 313		if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
 314			goto fail_unlock;
 315	}
 316found:
 317	inet_sk(sk)->inet_num = snum;
 318	udp_sk(sk)->udp_port_hash = snum;
 319	udp_sk(sk)->udp_portaddr_hash ^= snum;
 320	if (sk_unhashed(sk)) {
 321		if (sk->sk_reuseport &&
 322		    udp_reuseport_add_sock(sk, hslot)) {
 323			inet_sk(sk)->inet_num = 0;
 324			udp_sk(sk)->udp_port_hash = 0;
 325			udp_sk(sk)->udp_portaddr_hash ^= snum;
 326			goto fail_unlock;
 327		}
 328
 329		sock_set_flag(sk, SOCK_RCU_FREE);
 330
 331		sk_add_node_rcu(sk, &hslot->head);
 332		hslot->count++;
 333		sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 334
 335		hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
 336		spin_lock(&hslot2->lock);
 337		if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
 338		    sk->sk_family == AF_INET6)
 339			hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
 340					   &hslot2->head);
 341		else
 342			hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
 343					   &hslot2->head);
 344		hslot2->count++;
 345		spin_unlock(&hslot2->lock);
 346	}
 347
 348	error = 0;
 349fail_unlock:
 350	spin_unlock_bh(&hslot->lock);
 351fail:
 352	return error;
 353}
 354EXPORT_IPV6_MOD(udp_lib_get_port);
 355
 356int udp_v4_get_port(struct sock *sk, unsigned short snum)
 357{
 358	unsigned int hash2_nulladdr =
 359		ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
 360	unsigned int hash2_partial =
 361		ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
 362
 363	/* precompute partial secondary hash */
 364	udp_sk(sk)->udp_portaddr_hash = hash2_partial;
 365	return udp_lib_get_port(sk, snum, hash2_nulladdr);
 366}
 367
 368static int compute_score(struct sock *sk, const struct net *net,
 369			 __be32 saddr, __be16 sport,
 370			 __be32 daddr, unsigned short hnum,
 371			 int dif, int sdif)
 372{
 373	int score;
 374	struct inet_sock *inet;
 375	bool dev_match;
 376
 377	if (!net_eq(sock_net(sk), net) ||
 378	    udp_sk(sk)->udp_port_hash != hnum ||
 379	    ipv6_only_sock(sk))
 380		return -1;
 381
 382	if (sk->sk_rcv_saddr != daddr)
 383		return -1;
 384
 385	score = (sk->sk_family == PF_INET) ? 2 : 1;
 386
 387	inet = inet_sk(sk);
 388	if (inet->inet_daddr) {
 389		if (inet->inet_daddr != saddr)
 390			return -1;
 391		score += 4;
 392	}
 393
 394	if (inet->inet_dport) {
 395		if (inet->inet_dport != sport)
 396			return -1;
 397		score += 4;
 398	}
 399
 400	dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
 401					dif, sdif);
 402	if (!dev_match)
 403		return -1;
 404	if (sk->sk_bound_dev_if)
 405		score += 4;
 406
 407	if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
 408		score++;
 409	return score;
 410}
 411
 412u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
 413		const __be32 faddr, const __be16 fport)
 414{
 415	net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
 416
 417	return __inet_ehashfn(laddr, lport, faddr, fport,
 418			      udp_ehash_secret + net_hash_mix(net));
 419}
 420EXPORT_IPV6_MOD(udp_ehashfn);
 421
 422/**
 423 * udp4_lib_lookup1() - Simplified lookup using primary hash (destination port)
 424 * @net:	Network namespace
 425 * @saddr:	Source address, network order
 426 * @sport:	Source port, network order
 427 * @daddr:	Destination address, network order
 428 * @hnum:	Destination port, host order
 429 * @dif:	Destination interface index
 430 * @sdif:	Destination bridge port index, if relevant
 431 * @udptable:	Set of UDP hash tables
 432 *
 433 * Simplified lookup to be used as fallback if no sockets are found due to a
 434 * potential race between (receive) address change, and lookup happening before
 435 * the rehash operation. This function ignores SO_REUSEPORT groups while scoring
 436 * result sockets, because if we have one, we don't need the fallback at all.
 437 *
 438 * Called under rcu_read_lock().
 439 *
 440 * Return: socket with highest matching score if any, NULL if none
 441 */
 442static struct sock *udp4_lib_lookup1(const struct net *net,
 443				     __be32 saddr, __be16 sport,
 444				     __be32 daddr, unsigned int hnum,
 445				     int dif, int sdif,
 446				     const struct udp_table *udptable)
 447{
 448	unsigned int slot = udp_hashfn(net, hnum, udptable->mask);
 449	struct udp_hslot *hslot = &udptable->hash[slot];
 450	struct sock *sk, *result = NULL;
 451	int score, badness = 0;
 452
 453	sk_for_each_rcu(sk, &hslot->head) {
 454		score = compute_score(sk, net,
 455				      saddr, sport, daddr, hnum, dif, sdif);
 456		if (score > badness) {
 457			result = sk;
 458			badness = score;
 459		}
 460	}
 461
 462	return result;
 463}
 464
 465/* called with rcu_read_lock() */
 466static struct sock *udp4_lib_lookup2(const struct net *net,
 467				     __be32 saddr, __be16 sport,
 468				     __be32 daddr, unsigned int hnum,
 469				     int dif, int sdif,
 470				     struct udp_hslot *hslot2,
 471				     struct sk_buff *skb)
 472{
 473	struct sock *sk, *result;
 474	int score, badness;
 475	bool need_rescore;
 476
 477	result = NULL;
 478	badness = 0;
 479	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
 480		need_rescore = false;
 481rescore:
 482		score = compute_score(need_rescore ? result : sk, net, saddr,
 483				      sport, daddr, hnum, dif, sdif);
 484		if (score > badness) {
 485			badness = score;
 486
 487			if (need_rescore)
 488				continue;
 489
 490			if (sk->sk_state == TCP_ESTABLISHED) {
 491				result = sk;
 492				continue;
 493			}
 494
 495			result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
 496						       saddr, sport, daddr, hnum, udp_ehashfn);
 497			if (!result) {
 498				result = sk;
 499				continue;
 500			}
 501
 502			/* Fall back to scoring if group has connections */
 503			if (!reuseport_has_conns(sk))
 504				return result;
 505
 506			/* Reuseport logic returned an error, keep original score. */
 507			if (IS_ERR(result))
 508				continue;
 509
 510			/* compute_score is too long of a function to be
 511			 * inlined, and calling it again here yields
 512			 * measurable overhead for some
 513			 * workloads. Work around it by jumping
 514			 * backwards to rescore 'result'.
 515			 */
 516			need_rescore = true;
 517			goto rescore;
 518		}
 519	}
 520	return result;
 521}
 522
 523#if IS_ENABLED(CONFIG_BASE_SMALL)
 524static struct sock *udp4_lib_lookup4(const struct net *net,
 525				     __be32 saddr, __be16 sport,
 526				     __be32 daddr, unsigned int hnum,
 527				     int dif, int sdif,
 528				     struct udp_table *udptable)
 529{
 530	return NULL;
 531}
 532
 533static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
 534			u16 newhash4)
 535{
 536}
 537
 538static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
 539{
 540}
 541#else /* !CONFIG_BASE_SMALL */
 542static struct sock *udp4_lib_lookup4(const struct net *net,
 543				     __be32 saddr, __be16 sport,
 544				     __be32 daddr, unsigned int hnum,
 545				     int dif, int sdif,
 546				     struct udp_table *udptable)
 547{
 548	const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
 549	const struct hlist_nulls_node *node;
 550	struct udp_hslot *hslot4;
 551	unsigned int hash4, slot;
 552	struct udp_sock *up;
 553	struct sock *sk;
 554
 555	hash4 = udp_ehashfn(net, daddr, hnum, saddr, sport);
 556	slot = hash4 & udptable->mask;
 557	hslot4 = &udptable->hash4[slot];
 558	INET_ADDR_COOKIE(acookie, saddr, daddr);
 559
 560begin:
 561	/* SLAB_TYPESAFE_BY_RCU not used, so we don't need to touch sk_refcnt */
 562	udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
 563		sk = (struct sock *)up;
 564		if (inet_match(net, sk, acookie, ports, dif, sdif))
 565			return sk;
 566	}
 567
 568	/* if the nulls value we got at the end of this lookup is not the
 569	 * expected one, we must restart lookup. We probably met an item that
 570	 * was moved to another chain due to rehash.
 571	 */
 572	if (get_nulls_value(node) != slot)
 573		goto begin;
 574
 575	return NULL;
 576}
 577
 578/* udp_rehash4() only checks hslot4, and hash4_cnt is not processed. */
 579static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
 580			u16 newhash4)
 581{
 582	struct udp_hslot *hslot4, *nhslot4;
 583
 584	hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
 585	nhslot4 = udp_hashslot4(udptable, newhash4);
 586	udp_sk(sk)->udp_lrpa_hash = newhash4;
 587
 588	if (hslot4 != nhslot4) {
 589		spin_lock_bh(&hslot4->lock);
 590		hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
 591		hslot4->count--;
 592		spin_unlock_bh(&hslot4->lock);
 593
 594		spin_lock_bh(&nhslot4->lock);
 595		hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
 596					 &nhslot4->nulls_head);
 597		nhslot4->count++;
 598		spin_unlock_bh(&nhslot4->lock);
 599	}
 600}
 601
 602static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
 603{
 604	struct udp_hslot *hslot2, *hslot4;
 605
 606	if (udp_hashed4(sk)) {
 607		hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
 608		hslot4 = udp_hashslot4(udptable, udp_sk(sk)->udp_lrpa_hash);
 609
 610		spin_lock(&hslot4->lock);
 611		hlist_nulls_del_init_rcu(&udp_sk(sk)->udp_lrpa_node);
 612		hslot4->count--;
 613		spin_unlock(&hslot4->lock);
 614
 615		spin_lock(&hslot2->lock);
 616		udp_hash4_dec(hslot2);
 617		spin_unlock(&hslot2->lock);
 618	}
 619}
 620
 621void udp_lib_hash4(struct sock *sk, u16 hash)
 622{
 623	struct udp_hslot *hslot, *hslot2, *hslot4;
 624	struct net *net = sock_net(sk);
 625	struct udp_table *udptable;
 626
 627	/* Connected udp socket can re-connect to another remote address, which
 628	 * will be handled by rehash. Thus no need to redo hash4 here.
 629	 */
 630	if (udp_hashed4(sk))
 631		return;
 632
 633	udptable = net->ipv4.udp_table;
 634	hslot = udp_hashslot(udptable, net, udp_sk(sk)->udp_port_hash);
 635	hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
 636	hslot4 = udp_hashslot4(udptable, hash);
 637	udp_sk(sk)->udp_lrpa_hash = hash;
 638
 639	spin_lock_bh(&hslot->lock);
 640	if (rcu_access_pointer(sk->sk_reuseport_cb))
 641		reuseport_detach_sock(sk);
 642
 643	spin_lock(&hslot4->lock);
 644	hlist_nulls_add_head_rcu(&udp_sk(sk)->udp_lrpa_node,
 645				 &hslot4->nulls_head);
 646	hslot4->count++;
 647	spin_unlock(&hslot4->lock);
 648
 649	spin_lock(&hslot2->lock);
 650	udp_hash4_inc(hslot2);
 651	spin_unlock(&hslot2->lock);
 652
 653	spin_unlock_bh(&hslot->lock);
 654}
 655EXPORT_IPV6_MOD(udp_lib_hash4);
 656
 657/* call with sock lock */
 658void udp4_hash4(struct sock *sk)
 659{
 660	struct net *net = sock_net(sk);
 661	unsigned int hash;
 662
 663	if (sk_unhashed(sk) || sk->sk_rcv_saddr == htonl(INADDR_ANY))
 664		return;
 665
 666	hash = udp_ehashfn(net, sk->sk_rcv_saddr, sk->sk_num,
 667			   sk->sk_daddr, sk->sk_dport);
 668
 669	udp_lib_hash4(sk, hash);
 670}
 671EXPORT_IPV6_MOD(udp4_hash4);
 672#endif /* CONFIG_BASE_SMALL */
 673
 674/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
 675 * harder than this. -DaveM
 676 */
 677struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
 678		__be16 sport, __be32 daddr, __be16 dport, int dif,
 679		int sdif, struct udp_table *udptable, struct sk_buff *skb)
 680{
 681	unsigned short hnum = ntohs(dport);
 682	struct udp_hslot *hslot2;
 683	struct sock *result, *sk;
 684	unsigned int hash2;
 685
 686	hash2 = ipv4_portaddr_hash(net, daddr, hnum);
 687	hslot2 = udp_hashslot2(udptable, hash2);
 688
 689	if (udp_has_hash4(hslot2)) {
 690		result = udp4_lib_lookup4(net, saddr, sport, daddr, hnum,
 691					  dif, sdif, udptable);
 692		if (result) /* udp4_lib_lookup4 return sk or NULL */
 693			return result;
 694	}
 695
 696	/* Lookup connected or non-wildcard socket */
 697	result = udp4_lib_lookup2(net, saddr, sport,
 698				  daddr, hnum, dif, sdif,
 699				  hslot2, skb);
 700	if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
 701		goto done;
 702
 703	/* Lookup redirect from BPF */
 704	if (static_branch_unlikely(&bpf_sk_lookup_enabled) &&
 705	    udptable == net->ipv4.udp_table) {
 706		sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, sizeof(struct udphdr),
 707					       saddr, sport, daddr, hnum, dif,
 708					       udp_ehashfn);
 709		if (sk) {
 710			result = sk;
 711			goto done;
 712		}
 713	}
 714
 715	/* Got non-wildcard socket or error on first lookup */
 716	if (result)
 717		goto done;
 718
 719	/* Lookup wildcard sockets */
 720	hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
 721	hslot2 = udp_hashslot2(udptable, hash2);
 722
 723	result = udp4_lib_lookup2(net, saddr, sport,
 724				  htonl(INADDR_ANY), hnum, dif, sdif,
 725				  hslot2, skb);
 726	if (!IS_ERR_OR_NULL(result))
 727		goto done;
 728
 729	/* Primary hash (destination port) lookup as fallback for this race:
 730	 *   1. __ip4_datagram_connect() sets sk_rcv_saddr
 731	 *   2. lookup (this function): new sk_rcv_saddr, hashes not updated yet
 732	 *   3. rehash operation updating _secondary and four-tuple_ hashes
 733	 * The primary hash doesn't need an update after 1., so, thanks to this
 734	 * further step, 1. and 3. don't need to be atomic against the lookup.
 735	 */
 736	result = udp4_lib_lookup1(net, saddr, sport, daddr, hnum, dif, sdif,
 737				  udptable);
 738
 739done:
 740	if (IS_ERR(result))
 741		return NULL;
 742	return result;
 743}
 744EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
 745
 746static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
 747						 __be16 sport, __be16 dport,
 748						 struct udp_table *udptable)
 749{
 750	const struct iphdr *iph = ip_hdr(skb);
 751
 752	return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
 753				 iph->daddr, dport, inet_iif(skb),
 754				 inet_sdif(skb), udptable, skb);
 755}
 756
 757struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
 758				 __be16 sport, __be16 dport)
 759{
 760	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
 761	const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
 762	struct net *net = dev_net(skb->dev);
 763	int iif, sdif;
 764
 765	inet_get_iif_sdif(skb, &iif, &sdif);
 766
 767	return __udp4_lib_lookup(net, iph->saddr, sport,
 768				 iph->daddr, dport, iif,
 769				 sdif, net->ipv4.udp_table, NULL);
 770}
 771
 772/* Must be called under rcu_read_lock().
 773 * Does increment socket refcount.
 774 */
 775#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
 776struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
 777			     __be32 daddr, __be16 dport, int dif)
 778{
 779	struct sock *sk;
 780
 781	sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
 782			       dif, 0, net->ipv4.udp_table, NULL);
 783	if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
 784		sk = NULL;
 785	return sk;
 786}
 787EXPORT_SYMBOL_GPL(udp4_lib_lookup);
 788#endif
 789
 790static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
 791				       __be16 loc_port, __be32 loc_addr,
 792				       __be16 rmt_port, __be32 rmt_addr,
 793				       int dif, int sdif, unsigned short hnum)
 794{
 795	const struct inet_sock *inet = inet_sk(sk);
 796
 797	if (!net_eq(sock_net(sk), net) ||
 798	    udp_sk(sk)->udp_port_hash != hnum ||
 799	    (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
 800	    (inet->inet_dport != rmt_port && inet->inet_dport) ||
 801	    (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
 802	    ipv6_only_sock(sk) ||
 803	    !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
 804		return false;
 805	if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
 806		return false;
 807	return true;
 808}
 809
 810DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
 811EXPORT_IPV6_MOD(udp_encap_needed_key);
 812
 813#if IS_ENABLED(CONFIG_IPV6)
 814DEFINE_STATIC_KEY_FALSE(udpv6_encap_needed_key);
 815EXPORT_IPV6_MOD(udpv6_encap_needed_key);
 816#endif
 817
 818void udp_encap_enable(void)
 819{
 820	static_branch_inc(&udp_encap_needed_key);
 821}
 822EXPORT_SYMBOL(udp_encap_enable);
 823
 824void udp_encap_disable(void)
 825{
 826	static_branch_dec(&udp_encap_needed_key);
 827}
 828EXPORT_SYMBOL(udp_encap_disable);
 829
 830/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
 831 * through error handlers in encapsulations looking for a match.
 832 */
 833static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
 834{
 835	int i;
 836
 837	for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
 838		int (*handler)(struct sk_buff *skb, u32 info);
 839		const struct ip_tunnel_encap_ops *encap;
 840
 841		encap = rcu_dereference(iptun_encaps[i]);
 842		if (!encap)
 843			continue;
 844		handler = encap->err_handler;
 845		if (handler && !handler(skb, info))
 846			return 0;
 847	}
 848
 849	return -ENOENT;
 850}
 851
 852/* Try to match ICMP errors to UDP tunnels by looking up a socket without
 853 * reversing source and destination port: this will match tunnels that force the
 854 * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
 855 * lwtunnels might actually break this assumption by being configured with
 856 * different destination ports on endpoints, in this case we won't be able to
 857 * trace ICMP messages back to them.
 858 *
 859 * If this doesn't match any socket, probe tunnels with arbitrary destination
 860 * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
 861 * we've sent packets to won't necessarily match the local destination port.
 862 *
 863 * Then ask the tunnel implementation to match the error against a valid
 864 * association.
 865 *
 866 * Return an error if we can't find a match, the socket if we need further
 867 * processing, zero otherwise.
 868 */
 869static struct sock *__udp4_lib_err_encap(struct net *net,
 870					 const struct iphdr *iph,
 871					 struct udphdr *uh,
 872					 struct udp_table *udptable,
 873					 struct sock *sk,
 874					 struct sk_buff *skb, u32 info)
 875{
 876	int (*lookup)(struct sock *sk, struct sk_buff *skb);
 877	int network_offset, transport_offset;
 878	struct udp_sock *up;
 879
 880	network_offset = skb_network_offset(skb);
 881	transport_offset = skb_transport_offset(skb);
 882
 883	/* Network header needs to point to the outer IPv4 header inside ICMP */
 884	skb_reset_network_header(skb);
 885
 886	/* Transport header needs to point to the UDP header */
 887	skb_set_transport_header(skb, iph->ihl << 2);
 888
 889	if (sk) {
 890		up = udp_sk(sk);
 891
 892		lookup = READ_ONCE(up->encap_err_lookup);
 893		if (lookup && lookup(sk, skb))
 894			sk = NULL;
 895
 896		goto out;
 897	}
 898
 899	sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
 900			       iph->saddr, uh->dest, skb->dev->ifindex, 0,
 901			       udptable, NULL);
 902	if (sk) {
 903		up = udp_sk(sk);
 904
 905		lookup = READ_ONCE(up->encap_err_lookup);
 906		if (!lookup || lookup(sk, skb))
 907			sk = NULL;
 908	}
 909
 910out:
 911	if (!sk)
 912		sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
 913
 914	skb_set_transport_header(skb, transport_offset);
 915	skb_set_network_header(skb, network_offset);
 916
 917	return sk;
 918}
 919
 920/*
 921 * This routine is called by the ICMP module when it gets some
 922 * sort of error condition.  If err < 0 then the socket should
 923 * be closed and the error returned to the user.  If err > 0
 924 * it's just the icmp type << 8 | icmp code.
 925 * Header points to the ip header of the error packet. We move
 926 * on past this. Then (as it used to claim before adjustment)
 927 * header points to the first 8 bytes of the udp header.  We need
 928 * to find the appropriate port.
 929 */
 930
 931int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
 932{
 933	struct inet_sock *inet;
 934	const struct iphdr *iph = (const struct iphdr *)skb->data;
 935	struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
 936	const int type = icmp_hdr(skb)->type;
 937	const int code = icmp_hdr(skb)->code;
 938	bool tunnel = false;
 939	struct sock *sk;
 940	int harderr;
 941	int err;
 942	struct net *net = dev_net(skb->dev);
 943
 944	sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
 945			       iph->saddr, uh->source, skb->dev->ifindex,
 946			       inet_sdif(skb), udptable, NULL);
 947
 948	if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
 949		/* No socket for error: try tunnels before discarding */
 950		if (static_branch_unlikely(&udp_encap_needed_key)) {
 951			sk = __udp4_lib_err_encap(net, iph, uh, udptable, sk, skb,
 952						  info);
 953			if (!sk)
 954				return 0;
 955		} else
 956			sk = ERR_PTR(-ENOENT);
 957
 958		if (IS_ERR(sk)) {
 959			__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
 960			return PTR_ERR(sk);
 961		}
 962
 963		tunnel = true;
 964	}
 965
 966	err = 0;
 967	harderr = 0;
 968	inet = inet_sk(sk);
 969
 970	switch (type) {
 971	default:
 972	case ICMP_TIME_EXCEEDED:
 973		err = EHOSTUNREACH;
 974		break;
 975	case ICMP_SOURCE_QUENCH:
 976		goto out;
 977	case ICMP_PARAMETERPROB:
 978		err = EPROTO;
 979		harderr = 1;
 980		break;
 981	case ICMP_DEST_UNREACH:
 982		if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
 983			ipv4_sk_update_pmtu(skb, sk, info);
 984			if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
 985				err = EMSGSIZE;
 986				harderr = 1;
 987				break;
 988			}
 989			goto out;
 990		}
 991		err = EHOSTUNREACH;
 992		if (code <= NR_ICMP_UNREACH) {
 993			harderr = icmp_err_convert[code].fatal;
 994			err = icmp_err_convert[code].errno;
 995		}
 996		break;
 997	case ICMP_REDIRECT:
 998		ipv4_sk_redirect(skb, sk);
 999		goto out;
1000	}
1001
1002	/*
1003	 *      RFC1122: OK.  Passes ICMP errors back to application, as per
1004	 *	4.1.3.3.
1005	 */
1006	if (tunnel) {
1007		/* ...not for tunnels though: we don't have a sending socket */
1008		if (udp_sk(sk)->encap_err_rcv)
1009			udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
1010						  (u8 *)(uh+1));
1011		goto out;
1012	}
1013	if (!inet_test_bit(RECVERR, sk)) {
1014		if (!harderr || sk->sk_state != TCP_ESTABLISHED)
1015			goto out;
1016	} else
1017		ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
1018
1019	sk->sk_err = err;
1020	sk_error_report(sk);
1021out:
1022	return 0;
1023}
1024
1025int udp_err(struct sk_buff *skb, u32 info)
1026{
1027	return __udp4_lib_err(skb, info, dev_net(skb->dev)->ipv4.udp_table);
1028}
1029
1030/*
1031 * Throw away all pending data and cancel the corking. Socket is locked.
1032 */
1033void udp_flush_pending_frames(struct sock *sk)
1034{
1035	struct udp_sock *up = udp_sk(sk);
1036
1037	if (up->pending) {
1038		up->len = 0;
1039		WRITE_ONCE(up->pending, 0);
1040		ip_flush_pending_frames(sk);
1041	}
1042}
1043EXPORT_IPV6_MOD(udp_flush_pending_frames);
1044
1045/**
1046 * 	udp4_hwcsum  -  handle outgoing HW checksumming
1047 * 	@skb: 	sk_buff containing the filled-in UDP header
1048 * 	        (checksum field must be zeroed out)
1049 *	@src:	source IP address
1050 *	@dst:	destination IP address
1051 */
1052void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
1053{
1054	struct udphdr *uh = udp_hdr(skb);
1055	int offset = skb_transport_offset(skb);
1056	int len = skb->len - offset;
1057	int hlen = len;
1058	__wsum csum = 0;
1059
1060	if (!skb_has_frag_list(skb)) {
1061		/*
1062		 * Only one fragment on the socket.
1063		 */
1064		skb->csum_start = skb_transport_header(skb) - skb->head;
1065		skb->csum_offset = offsetof(struct udphdr, check);
1066		uh->check = ~csum_tcpudp_magic(src, dst, len,
1067					       IPPROTO_UDP, 0);
1068	} else {
1069		struct sk_buff *frags;
1070
1071		/*
1072		 * HW-checksum won't work as there are two or more
1073		 * fragments on the socket so that all csums of sk_buffs
1074		 * should be together
1075		 */
1076		skb_walk_frags(skb, frags) {
1077			csum = csum_add(csum, frags->csum);
1078			hlen -= frags->len;
1079		}
1080
1081		csum = skb_checksum(skb, offset, hlen, csum);
1082		skb->ip_summed = CHECKSUM_NONE;
1083
1084		uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
1085		if (uh->check == 0)
1086			uh->check = CSUM_MANGLED_0;
1087	}
1088}
1089EXPORT_SYMBOL_GPL(udp4_hwcsum);
1090
1091/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
1092 * for the simple case like when setting the checksum for a UDP tunnel.
1093 */
1094void udp_set_csum(bool nocheck, struct sk_buff *skb,
1095		  __be32 saddr, __be32 daddr, int len)
1096{
1097	struct udphdr *uh = udp_hdr(skb);
1098
1099	if (nocheck) {
1100		uh->check = 0;
1101	} else if (skb_is_gso(skb)) {
1102		uh->check = ~udp_v4_check(len, saddr, daddr, 0);
1103	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
1104		uh->check = 0;
1105		uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
1106		if (uh->check == 0)
1107			uh->check = CSUM_MANGLED_0;
1108	} else {
1109		skb->ip_summed = CHECKSUM_PARTIAL;
1110		skb->csum_start = skb_transport_header(skb) - skb->head;
1111		skb->csum_offset = offsetof(struct udphdr, check);
1112		uh->check = ~udp_v4_check(len, saddr, daddr, 0);
1113	}
1114}
1115EXPORT_SYMBOL(udp_set_csum);
1116
1117static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
1118			struct inet_cork *cork)
1119{
1120	struct sock *sk = skb->sk;
1121	struct inet_sock *inet = inet_sk(sk);
1122	struct udphdr *uh;
1123	int err;
1124	int is_udplite = IS_UDPLITE(sk);
1125	int offset = skb_transport_offset(skb);
1126	int len = skb->len - offset;
1127	int datalen = len - sizeof(*uh);
1128	__wsum csum = 0;
1129
1130	/*
1131	 * Create a UDP header
1132	 */
1133	uh = udp_hdr(skb);
1134	uh->source = inet->inet_sport;
1135	uh->dest = fl4->fl4_dport;
1136	uh->len = htons(len);
1137	uh->check = 0;
1138
1139	if (cork->gso_size) {
1140		const int hlen = skb_network_header_len(skb) +
1141				 sizeof(struct udphdr);
1142
1143		if (hlen + min(datalen, cork->gso_size) > cork->fragsize) {
1144			kfree_skb(skb);
1145			return -EMSGSIZE;
1146		}
1147		if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
1148			kfree_skb(skb);
1149			return -EINVAL;
1150		}
1151		if (sk->sk_no_check_tx) {
1152			kfree_skb(skb);
1153			return -EINVAL;
1154		}
1155		if (is_udplite || dst_xfrm(skb_dst(skb))) {
1156			kfree_skb(skb);
1157			return -EIO;
1158		}
1159
1160		if (datalen > cork->gso_size) {
1161			skb_shinfo(skb)->gso_size = cork->gso_size;
1162			skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
1163			skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
1164								 cork->gso_size);
1165
1166			/* Don't checksum the payload, skb will get segmented */
1167			goto csum_partial;
1168		}
1169	}
1170
1171	if (is_udplite)  				 /*     UDP-Lite      */
1172		csum = udplite_csum(skb);
1173
1174	else if (sk->sk_no_check_tx) {			 /* UDP csum off */
1175
1176		skb->ip_summed = CHECKSUM_NONE;
1177		goto send;
1178
1179	} else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
1180csum_partial:
1181
1182		udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
1183		goto send;
1184
1185	} else
1186		csum = udp_csum(skb);
1187
1188	/* add protocol-dependent pseudo-header */
1189	uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
1190				      sk->sk_protocol, csum);
1191	if (uh->check == 0)
1192		uh->check = CSUM_MANGLED_0;
1193
1194send:
1195	err = ip_send_skb(sock_net(sk), skb);
1196	if (err) {
1197		if (err == -ENOBUFS &&
1198		    !inet_test_bit(RECVERR, sk)) {
1199			UDP_INC_STATS(sock_net(sk),
1200				      UDP_MIB_SNDBUFERRORS, is_udplite);
1201			err = 0;
1202		}
1203	} else
1204		UDP_INC_STATS(sock_net(sk),
1205			      UDP_MIB_OUTDATAGRAMS, is_udplite);
1206	return err;
1207}
1208
1209/*
1210 * Push out all pending data as one UDP datagram. Socket is locked.
1211 */
1212int udp_push_pending_frames(struct sock *sk)
1213{
1214	struct udp_sock  *up = udp_sk(sk);
1215	struct inet_sock *inet = inet_sk(sk);
1216	struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
1217	struct sk_buff *skb;
1218	int err = 0;
1219
1220	skb = ip_finish_skb(sk, fl4);
1221	if (!skb)
1222		goto out;
1223
1224	err = udp_send_skb(skb, fl4, &inet->cork.base);
1225
1226out:
1227	up->len = 0;
1228	WRITE_ONCE(up->pending, 0);
1229	return err;
1230}
1231EXPORT_IPV6_MOD(udp_push_pending_frames);
1232
1233static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
1234{
1235	switch (cmsg->cmsg_type) {
1236	case UDP_SEGMENT:
1237		if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
1238			return -EINVAL;
1239		*gso_size = *(__u16 *)CMSG_DATA(cmsg);
1240		return 0;
1241	default:
1242		return -EINVAL;
1243	}
1244}
1245
1246int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
1247{
1248	struct cmsghdr *cmsg;
1249	bool need_ip = false;
1250	int err;
1251
1252	for_each_cmsghdr(cmsg, msg) {
1253		if (!CMSG_OK(msg, cmsg))
1254			return -EINVAL;
1255
1256		if (cmsg->cmsg_level != SOL_UDP) {
1257			need_ip = true;
1258			continue;
1259		}
1260
1261		err = __udp_cmsg_send(cmsg, gso_size);
1262		if (err)
1263			return err;
1264	}
1265
1266	return need_ip;
1267}
1268EXPORT_IPV6_MOD_GPL(udp_cmsg_send);
1269
1270int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1271{
1272	struct inet_sock *inet = inet_sk(sk);
1273	struct udp_sock *up = udp_sk(sk);
1274	DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
1275	struct flowi4 fl4_stack;
1276	struct flowi4 *fl4;
1277	int ulen = len;
1278	struct ipcm_cookie ipc;
1279	struct rtable *rt = NULL;
1280	int free = 0;
1281	int connected = 0;
1282	__be32 daddr, faddr, saddr;
1283	u8 scope;
1284	__be16 dport;
1285	int err, is_udplite = IS_UDPLITE(sk);
1286	int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
1287	int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
1288	struct sk_buff *skb;
1289	struct ip_options_data opt_copy;
1290	int uc_index;
1291
1292	if (len > 0xFFFF)
1293		return -EMSGSIZE;
1294
1295	/*
1296	 *	Check the flags.
1297	 */
1298
1299	if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
1300		return -EOPNOTSUPP;
1301
1302	getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
1303
1304	fl4 = &inet->cork.fl.u.ip4;
1305	if (READ_ONCE(up->pending)) {
1306		/*
1307		 * There are pending frames.
1308		 * The socket lock must be held while it's corked.
1309		 */
1310		lock_sock(sk);
1311		if (likely(up->pending)) {
1312			if (unlikely(up->pending != AF_INET)) {
1313				release_sock(sk);
1314				return -EINVAL;
1315			}
1316			goto do_append_data;
1317		}
1318		release_sock(sk);
1319	}
1320	ulen += sizeof(struct udphdr);
1321
1322	/*
1323	 *	Get and verify the address.
1324	 */
1325	if (usin) {
1326		if (msg->msg_namelen < sizeof(*usin))
1327			return -EINVAL;
1328		if (usin->sin_family != AF_INET) {
1329			if (usin->sin_family != AF_UNSPEC)
1330				return -EAFNOSUPPORT;
1331		}
1332
1333		daddr = usin->sin_addr.s_addr;
1334		dport = usin->sin_port;
1335		if (dport == 0)
1336			return -EINVAL;
1337	} else {
1338		if (sk->sk_state != TCP_ESTABLISHED)
1339			return -EDESTADDRREQ;
1340		daddr = inet->inet_daddr;
1341		dport = inet->inet_dport;
1342		/* Open fast path for connected socket.
1343		   Route will not be used, if at least one option is set.
1344		 */
1345		connected = 1;
1346	}
1347
1348	ipcm_init_sk(&ipc, inet);
1349	ipc.gso_size = READ_ONCE(up->gso_size);
1350
1351	if (msg->msg_controllen) {
1352		err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1353		if (err > 0) {
1354			err = ip_cmsg_send(sk, msg, &ipc,
1355					   sk->sk_family == AF_INET6);
1356			connected = 0;
1357		}
1358		if (unlikely(err < 0)) {
1359			kfree(ipc.opt);
1360			return err;
1361		}
1362		if (ipc.opt)
1363			free = 1;
1364	}
1365	if (!ipc.opt) {
1366		struct ip_options_rcu *inet_opt;
1367
1368		rcu_read_lock();
1369		inet_opt = rcu_dereference(inet->inet_opt);
1370		if (inet_opt) {
1371			memcpy(&opt_copy, inet_opt,
1372			       sizeof(*inet_opt) + inet_opt->opt.optlen);
1373			ipc.opt = &opt_copy.opt;
1374		}
1375		rcu_read_unlock();
1376	}
1377
1378	if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
1379		err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1380					    (struct sockaddr *)usin,
1381					    &msg->msg_namelen,
1382					    &ipc.addr);
1383		if (err)
1384			goto out_free;
1385		if (usin) {
1386			if (usin->sin_port == 0) {
1387				/* BPF program set invalid port. Reject it. */
1388				err = -EINVAL;
1389				goto out_free;
1390			}
1391			daddr = usin->sin_addr.s_addr;
1392			dport = usin->sin_port;
1393		}
1394	}
1395
1396	saddr = ipc.addr;
1397	ipc.addr = faddr = daddr;
1398
1399	if (ipc.opt && ipc.opt->opt.srr) {
1400		if (!daddr) {
1401			err = -EINVAL;
1402			goto out_free;
1403		}
1404		faddr = ipc.opt->opt.faddr;
1405		connected = 0;
1406	}
1407	scope = ip_sendmsg_scope(inet, &ipc, msg);
1408	if (scope == RT_SCOPE_LINK)
1409		connected = 0;
1410
1411	uc_index = READ_ONCE(inet->uc_index);
1412	if (ipv4_is_multicast(daddr)) {
1413		if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
1414			ipc.oif = READ_ONCE(inet->mc_index);
1415		if (!saddr)
1416			saddr = READ_ONCE(inet->mc_addr);
1417		connected = 0;
1418	} else if (!ipc.oif) {
1419		ipc.oif = uc_index;
1420	} else if (ipv4_is_lbcast(daddr) && uc_index) {
1421		/* oif is set, packet is to local broadcast and
1422		 * uc_index is set. oif is most likely set
1423		 * by sk_bound_dev_if. If uc_index != oif check if the
1424		 * oif is an L3 master and uc_index is an L3 slave.
1425		 * If so, we want to allow the send using the uc_index.
1426		 */
1427		if (ipc.oif != uc_index &&
1428		    ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
1429							      uc_index)) {
1430			ipc.oif = uc_index;
1431		}
1432	}
1433
1434	if (connected)
1435		rt = dst_rtable(sk_dst_check(sk, 0));
1436
1437	if (!rt) {
1438		struct net *net = sock_net(sk);
1439		__u8 flow_flags = inet_sk_flowi_flags(sk);
1440
1441		fl4 = &fl4_stack;
1442
1443		flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark,
1444				   ipc.tos & INET_DSCP_MASK, scope,
1445				   sk->sk_protocol, flow_flags, faddr, saddr,
1446				   dport, inet->inet_sport,
1447				   sk_uid(sk));
1448
1449		security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
1450		rt = ip_route_output_flow(net, fl4, sk);
1451		if (IS_ERR(rt)) {
1452			err = PTR_ERR(rt);
1453			rt = NULL;
1454			if (err == -ENETUNREACH)
1455				IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
1456			goto out;
1457		}
1458
1459		err = -EACCES;
1460		if ((rt->rt_flags & RTCF_BROADCAST) &&
1461		    !sock_flag(sk, SOCK_BROADCAST))
1462			goto out;
1463		if (connected)
1464			sk_dst_set(sk, dst_clone(&rt->dst));
1465	}
1466
1467	if (msg->msg_flags&MSG_CONFIRM)
1468		goto do_confirm;
1469back_from_confirm:
1470
1471	saddr = fl4->saddr;
1472	if (!ipc.addr)
1473		daddr = ipc.addr = fl4->daddr;
1474
1475	/* Lockless fast path for the non-corking case. */
1476	if (!corkreq) {
1477		struct inet_cork cork;
1478
1479		skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
1480				  sizeof(struct udphdr), &ipc, &rt,
1481				  &cork, msg->msg_flags);
1482		err = PTR_ERR(skb);
1483		if (!IS_ERR_OR_NULL(skb))
1484			err = udp_send_skb(skb, fl4, &cork);
1485		goto out;
1486	}
1487
1488	lock_sock(sk);
1489	if (unlikely(up->pending)) {
1490		/* The socket is already corked while preparing it. */
1491		/* ... which is an evident application bug. --ANK */
1492		release_sock(sk);
1493
1494		net_dbg_ratelimited("socket already corked\n");
1495		err = -EINVAL;
1496		goto out;
1497	}
1498	/*
1499	 *	Now cork the socket to pend data.
1500	 */
1501	fl4 = &inet->cork.fl.u.ip4;
1502	fl4->daddr = daddr;
1503	fl4->saddr = saddr;
1504	fl4->fl4_dport = dport;
1505	fl4->fl4_sport = inet->inet_sport;
1506	WRITE_ONCE(up->pending, AF_INET);
1507
1508do_append_data:
1509	up->len += ulen;
1510	err = ip_append_data(sk, fl4, getfrag, msg, ulen,
1511			     sizeof(struct udphdr), &ipc, &rt,
1512			     corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
1513	if (err)
1514		udp_flush_pending_frames(sk);
1515	else if (!corkreq)
1516		err = udp_push_pending_frames(sk);
1517	else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1518		WRITE_ONCE(up->pending, 0);
1519	release_sock(sk);
1520
1521out:
1522	ip_rt_put(rt);
1523out_free:
1524	if (free)
1525		kfree(ipc.opt);
1526	if (!err)
1527		return len;
1528	/*
1529	 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space.  Reporting
1530	 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1531	 * we don't have a good statistic (IpOutDiscards but it can be too many
1532	 * things).  We could add another new stat but at least for now that
1533	 * seems like overkill.
1534	 */
1535	if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1536		UDP_INC_STATS(sock_net(sk),
1537			      UDP_MIB_SNDBUFERRORS, is_udplite);
1538	}
1539	return err;
1540
1541do_confirm:
1542	if (msg->msg_flags & MSG_PROBE)
1543		dst_confirm_neigh(&rt->dst, &fl4->daddr);
1544	if (!(msg->msg_flags&MSG_PROBE) || len)
1545		goto back_from_confirm;
1546	err = 0;
1547	goto out;
1548}
1549EXPORT_SYMBOL(udp_sendmsg);
1550
1551void udp_splice_eof(struct socket *sock)
1552{
1553	struct sock *sk = sock->sk;
1554	struct udp_sock *up = udp_sk(sk);
1555
1556	if (!READ_ONCE(up->pending) || udp_test_bit(CORK, sk))
1557		return;
1558
1559	lock_sock(sk);
1560	if (up->pending && !udp_test_bit(CORK, sk))
1561		udp_push_pending_frames(sk);
1562	release_sock(sk);
1563}
1564EXPORT_IPV6_MOD_GPL(udp_splice_eof);
1565
1566#define UDP_SKB_IS_STATELESS 0x80000000
1567
1568/* all head states (dst, sk, nf conntrack) except skb extensions are
1569 * cleared by udp_rcv().
1570 *
1571 * We need to preserve secpath, if present, to eventually process
1572 * IP_CMSG_PASSSEC at recvmsg() time.
1573 *
1574 * Other extensions can be cleared.
1575 */
1576static bool udp_try_make_stateless(struct sk_buff *skb)
1577{
1578	if (!skb_has_extensions(skb))
1579		return true;
1580
1581	if (!secpath_exists(skb)) {
1582		skb_ext_reset(skb);
1583		return true;
1584	}
1585
1586	return false;
1587}
1588
1589static void udp_set_dev_scratch(struct sk_buff *skb)
1590{
1591	struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
1592
1593	BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
1594	scratch->_tsize_state = skb->truesize;
1595#if BITS_PER_LONG == 64
1596	scratch->len = skb->len;
1597	scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1598	scratch->is_linear = !skb_is_nonlinear(skb);
1599#endif
1600	if (udp_try_make_stateless(skb))
1601		scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
1602}
1603
1604static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
1605{
1606	/* We come here after udp_lib_checksum_complete() returned 0.
1607	 * This means that __skb_checksum_complete() might have
1608	 * set skb->csum_valid to 1.
1609	 * On 64bit platforms, we can set csum_unnecessary
1610	 * to true, but only if the skb is not shared.
1611	 */
1612#if BITS_PER_LONG == 64
1613	if (!skb_shared(skb))
1614		udp_skb_scratch(skb)->csum_unnecessary = true;
1615#endif
1616}
1617
1618static int udp_skb_truesize(struct sk_buff *skb)
1619{
1620	return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
1621}
1622
1623static bool udp_skb_has_head_state(struct sk_buff *skb)
1624{
1625	return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
1626}
1627
1628/* fully reclaim rmem/fwd memory allocated for skb */
1629static void udp_rmem_release(struct sock *sk, unsigned int size,
1630			     int partial, bool rx_queue_lock_held)
1631{
1632	struct udp_sock *up = udp_sk(sk);
1633	struct sk_buff_head *sk_queue;
1634	unsigned int amt;
1635
1636	if (likely(partial)) {
1637		up->forward_deficit += size;
1638		size = up->forward_deficit;
1639		if (size < READ_ONCE(up->forward_threshold) &&
1640		    !skb_queue_empty(&up->reader_queue))
1641			return;
1642	} else {
1643		size += up->forward_deficit;
1644	}
1645	up->forward_deficit = 0;
1646
1647	/* acquire the sk_receive_queue for fwd allocated memory scheduling,
1648	 * if the called don't held it already
1649	 */
1650	sk_queue = &sk->sk_receive_queue;
1651	if (!rx_queue_lock_held)
1652		spin_lock(&sk_queue->lock);
1653
1654	amt = (size + sk->sk_forward_alloc - partial) & ~(PAGE_SIZE - 1);
1655	sk_forward_alloc_add(sk, size - amt);
1656
1657	if (amt)
1658		__sk_mem_reduce_allocated(sk, amt >> PAGE_SHIFT);
1659
1660	atomic_sub(size, &sk->sk_rmem_alloc);
1661
1662	/* this can save us from acquiring the rx queue lock on next receive */
1663	skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
1664
1665	if (!rx_queue_lock_held)
1666		spin_unlock(&sk_queue->lock);
1667}
1668
1669/* Note: called with reader_queue.lock held.
1670 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1671 * This avoids a cache line miss while receive_queue lock is held.
1672 * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1673 */
1674void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
1675{
1676	prefetch(&skb->data);
1677	udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
1678}
1679EXPORT_IPV6_MOD(udp_skb_destructor);
1680
1681/* as above, but the caller held the rx queue lock, too */
1682static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
1683{
1684	prefetch(&skb->data);
1685	udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
1686}
1687
1688static int udp_rmem_schedule(struct sock *sk, int size)
1689{
1690	int delta;
1691
1692	delta = size - sk->sk_forward_alloc;
1693	if (delta > 0 && !__sk_mem_schedule(sk, delta, SK_MEM_RECV))
1694		return -ENOBUFS;
1695
1696	return 0;
1697}
1698
1699int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1700{
1701	struct sk_buff_head *list = &sk->sk_receive_queue;
1702	struct udp_prod_queue *udp_prod_queue;
1703	struct sk_buff *next, *to_drop = NULL;
1704	struct llist_node *ll_list;
1705	unsigned int rmem, rcvbuf;
1706	int size, err = -ENOMEM;
1707	int total_size = 0;
1708	int q_size = 0;
1709	int dropcount;
1710	int nb = 0;
1711
1712	rmem = atomic_read(&sk->sk_rmem_alloc);
1713	rcvbuf = READ_ONCE(sk->sk_rcvbuf);
1714	size = skb->truesize;
1715
1716	udp_prod_queue = &udp_sk(sk)->udp_prod_queue[numa_node_id()];
1717
1718	rmem += atomic_read(&udp_prod_queue->rmem_alloc);
1719
1720	/* Immediately drop when the receive queue is full.
1721	 * Cast to unsigned int performs the boundary check for INT_MAX.
1722	 */
1723	if (rmem + size > rcvbuf) {
1724		if (rcvbuf > INT_MAX >> 1)
1725			goto drop;
1726
1727		/* Accept the packet if queue is empty. */
1728		if (rmem)
1729			goto drop;
1730	}
1731
1732	/* Under mem pressure, it might be helpful to help udp_recvmsg()
1733	 * having linear skbs :
1734	 * - Reduce memory overhead and thus increase receive queue capacity
1735	 * - Less cache line misses at copyout() time
1736	 * - Less work at consume_skb() (less alien page frag freeing)
1737	 */
1738	if (rmem > (rcvbuf >> 1)) {
1739		skb_condense(skb);
1740		size = skb->truesize;
1741	}
1742
1743	udp_set_dev_scratch(skb);
1744
1745	atomic_add(size, &udp_prod_queue->rmem_alloc);
1746
1747	if (!llist_add(&skb->ll_node, &udp_prod_queue->ll_root))
1748		return 0;
1749
1750	dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ? sk_drops_read(sk) : 0;
1751
1752	spin_lock(&list->lock);
1753
1754	ll_list = llist_del_all(&udp_prod_queue->ll_root);
1755
1756	ll_list = llist_reverse_order(ll_list);
1757
1758	llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
1759		size = udp_skb_truesize(skb);
1760		total_size += size;
1761		err = udp_rmem_schedule(sk, size);
1762		if (unlikely(err)) {
1763			/*  Free the skbs outside of locked section. */
1764			skb->next = to_drop;
1765			to_drop = skb;
1766			continue;
1767		}
1768
1769		q_size += size;
1770		sk_forward_alloc_add(sk, -size);
1771
1772		/* no need to setup a destructor, we will explicitly release the
1773		 * forward allocated memory on dequeue
1774		 */
1775		SOCK_SKB_CB(skb)->dropcount = dropcount;
1776		nb++;
1777		__skb_queue_tail(list, skb);
1778	}
1779
1780	atomic_add(q_size, &sk->sk_rmem_alloc);
1781
1782	spin_unlock(&list->lock);
1783
1784	if (!sock_flag(sk, SOCK_DEAD)) {
1785		/* Multiple threads might be blocked in recvmsg(),
1786		 * using prepare_to_wait_exclusive().
1787		 */
1788		while (nb) {
1789			INDIRECT_CALL_1(READ_ONCE(sk->sk_data_ready),
1790					sock_def_readable, sk);
1791			nb--;
1792		}
1793	}
1794
1795	if (unlikely(to_drop)) {
1796		for (nb = 0; to_drop != NULL; nb++) {
1797			skb = to_drop;
1798			to_drop = skb->next;
1799			skb_mark_not_on_list(skb);
1800			/* TODO: update SNMP values. */
1801			sk_skb_reason_drop(sk, skb, SKB_DROP_REASON_PROTO_MEM);
1802		}
1803		numa_drop_add(&udp_sk(sk)->drop_counters, nb);
1804	}
1805
1806	atomic_sub(total_size, &udp_prod_queue->rmem_alloc);
1807
1808	return 0;
1809
1810drop:
1811	udp_drops_inc(sk);
1812	return err;
1813}
1814EXPORT_IPV6_MOD_GPL(__udp_enqueue_schedule_skb);
1815
1816void udp_destruct_common(struct sock *sk)
1817{
1818	/* reclaim completely the forward allocated memory */
1819	struct udp_sock *up = udp_sk(sk);
1820	unsigned int total = 0;
1821	struct sk_buff *skb;
1822
1823	skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
1824	while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
1825		total += skb->truesize;
1826		kfree_skb(skb);
1827	}
1828	udp_rmem_release(sk, total, 0, true);
1829	kfree(up->udp_prod_queue);
1830}
1831EXPORT_IPV6_MOD_GPL(udp_destruct_common);
1832
1833static void udp_destruct_sock(struct sock *sk)
1834{
1835	udp_destruct_common(sk);
1836	inet_sock_destruct(sk);
1837}
1838
1839int udp_init_sock(struct sock *sk)
1840{
1841	int res = udp_lib_init_sock(sk);
1842
1843	sk->sk_destruct = udp_destruct_sock;
1844	set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
1845	return res;
1846}
1847
1848void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1849{
1850	if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset)))
1851		sk_peek_offset_bwd(sk, len);
1852
1853	if (!skb_shared(skb)) {
1854		skb_orphan(skb);
1855		skb_attempt_defer_free(skb);
1856		return;
1857	}
1858
1859	if (!skb_unref(skb))
1860		return;
1861
1862	/* In the more common cases we cleared the head states previously,
1863	 * see __udp_queue_rcv_skb().
1864	 */
1865	if (unlikely(udp_skb_has_head_state(skb)))
1866		skb_release_head_state(skb);
1867	__consume_stateless_skb(skb);
1868}
1869EXPORT_IPV6_MOD_GPL(skb_consume_udp);
1870
1871static struct sk_buff *__first_packet_length(struct sock *sk,
1872					     struct sk_buff_head *rcvq,
1873					     unsigned int *total)
1874{
1875	struct sk_buff *skb;
1876
1877	while ((skb = skb_peek(rcvq)) != NULL) {
1878		if (udp_lib_checksum_complete(skb)) {
1879			__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1880					IS_UDPLITE(sk));
1881			__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1882					IS_UDPLITE(sk));
1883			udp_drops_inc(sk);
1884			__skb_unlink(skb, rcvq);
1885			*total += skb->truesize;
1886			kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
1887		} else {
1888			udp_skb_csum_unnecessary_set(skb);
1889			break;
1890		}
1891	}
1892	return skb;
1893}
1894
1895/**
1896 *	first_packet_length	- return length of first packet in receive queue
1897 *	@sk: socket
1898 *
1899 *	Drops all bad checksum frames, until a valid one is found.
1900 *	Returns the length of found skb, or -1 if none is found.
1901 */
1902static int first_packet_length(struct sock *sk)
1903{
1904	struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1905	struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1906	unsigned int total = 0;
1907	struct sk_buff *skb;
1908	int res;
1909
1910	spin_lock_bh(&rcvq->lock);
1911	skb = __first_packet_length(sk, rcvq, &total);
1912	if (!skb && !skb_queue_empty_lockless(sk_queue)) {
1913		spin_lock(&sk_queue->lock);
1914		skb_queue_splice_tail_init(sk_queue, rcvq);
1915		spin_unlock(&sk_queue->lock);
1916
1917		skb = __first_packet_length(sk, rcvq, &total);
1918	}
1919	res = skb ? skb->len : -1;
1920	if (total)
1921		udp_rmem_release(sk, total, 1, false);
1922	spin_unlock_bh(&rcvq->lock);
1923	return res;
1924}
1925
1926/*
1927 *	IOCTL requests applicable to the UDP protocol
1928 */
1929
1930int udp_ioctl(struct sock *sk, int cmd, int *karg)
1931{
1932	switch (cmd) {
1933	case SIOCOUTQ:
1934	{
1935		*karg = sk_wmem_alloc_get(sk);
1936		return 0;
1937	}
1938
1939	case SIOCINQ:
1940	{
1941		*karg = max_t(int, 0, first_packet_length(sk));
1942		return 0;
1943	}
1944
1945	default:
1946		return -ENOIOCTLCMD;
1947	}
1948
1949	return 0;
1950}
1951EXPORT_IPV6_MOD(udp_ioctl);
1952
1953struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
1954			       int *off, int *err)
1955{
1956	struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1957	struct sk_buff_head *queue;
1958	struct sk_buff *last;
1959	long timeo;
1960	int error;
1961
1962	queue = &udp_sk(sk)->reader_queue;
1963	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1964	do {
1965		struct sk_buff *skb;
1966
1967		error = sock_error(sk);
1968		if (error)
1969			break;
1970
1971		error = -EAGAIN;
1972		do {
1973			spin_lock_bh(&queue->lock);
1974			skb = __skb_try_recv_from_queue(queue, flags, off, err,
1975							&last);
1976			if (skb) {
1977				if (!(flags & MSG_PEEK))
1978					udp_skb_destructor(sk, skb);
1979				spin_unlock_bh(&queue->lock);
1980				return skb;
1981			}
1982
1983			if (skb_queue_empty_lockless(sk_queue)) {
1984				spin_unlock_bh(&queue->lock);
1985				goto busy_check;
1986			}
1987
1988			/* refill the reader queue and walk it again
1989			 * keep both queues locked to avoid re-acquiring
1990			 * the sk_receive_queue lock if fwd memory scheduling
1991			 * is needed.
1992			 */
1993			spin_lock(&sk_queue->lock);
1994			skb_queue_splice_tail_init(sk_queue, queue);
1995
1996			skb = __skb_try_recv_from_queue(queue, flags, off, err,
1997							&last);
1998			if (skb && !(flags & MSG_PEEK))
1999				udp_skb_dtor_locked(sk, skb);
2000			spin_unlock(&sk_queue->lock);
2001			spin_unlock_bh(&queue->lock);
2002			if (skb)
2003				return skb;
2004
2005busy_check:
2006			if (!sk_can_busy_loop(sk))
2007				break;
2008
2009			sk_busy_loop(sk, flags & MSG_DONTWAIT);
2010		} while (!skb_queue_empty_lockless(sk_queue));
2011
2012		/* sk_queue is empty, reader_queue may contain peeked packets */
2013	} while (timeo &&
2014		 !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
2015					      &error, &timeo,
2016					      (struct sk_buff *)sk_queue));
2017
2018	*err = error;
2019	return NULL;
2020}
2021EXPORT_SYMBOL(__skb_recv_udp);
2022
2023int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
2024{
2025	struct sk_buff *skb;
2026	int err;
2027
2028try_again:
2029	skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
2030	if (!skb)
2031		return err;
2032
2033	if (udp_lib_checksum_complete(skb)) {
2034		int is_udplite = IS_UDPLITE(sk);
2035		struct net *net = sock_net(sk);
2036
2037		__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, is_udplite);
2038		__UDP_INC_STATS(net, UDP_MIB_INERRORS, is_udplite);
2039		udp_drops_inc(sk);
2040		kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
2041		goto try_again;
2042	}
2043
2044	WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
2045	return recv_actor(sk, skb);
2046}
2047EXPORT_IPV6_MOD(udp_read_skb);
2048
2049/*
2050 * 	This should be easy, if there is something there we
2051 * 	return it, otherwise we block.
2052 */
2053
2054int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
2055		int *addr_len)
2056{
2057	struct inet_sock *inet = inet_sk(sk);
2058	DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
2059	struct sk_buff *skb;
2060	unsigned int ulen, copied;
2061	int off, err, peeking = flags & MSG_PEEK;
2062	int is_udplite = IS_UDPLITE(sk);
2063	bool checksum_valid = false;
2064
2065	if (flags & MSG_ERRQUEUE)
2066		return ip_recv_error(sk, msg, len, addr_len);
2067
2068try_again:
2069	off = sk_peek_offset(sk, flags);
2070	skb = __skb_recv_udp(sk, flags, &off, &err);
2071	if (!skb)
2072		return err;
2073
2074	ulen = udp_skb_len(skb);
2075	copied = len;
2076	if (copied > ulen - off)
2077		copied = ulen - off;
2078	else if (copied < ulen)
2079		msg->msg_flags |= MSG_TRUNC;
2080
2081	/*
2082	 * If checksum is needed at all, try to do it while copying the
2083	 * data.  If the data is truncated, or if we only want a partial
2084	 * coverage checksum (UDP-Lite), do it before the copy.
2085	 */
2086
2087	if (copied < ulen || peeking ||
2088	    (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
2089		checksum_valid = udp_skb_csum_unnecessary(skb) ||
2090				!__udp_lib_checksum_complete(skb);
2091		if (!checksum_valid)
2092			goto csum_copy_err;
2093	}
2094
2095	if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
2096		if (udp_skb_is_linear(skb))
2097			err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
2098		else
2099			err = skb_copy_datagram_msg(skb, off, msg, copied);
2100	} else {
2101		err = skb_copy_and_csum_datagram_msg(skb, off, msg);
2102
2103		if (err == -EINVAL)
2104			goto csum_copy_err;
2105	}
2106
2107	if (unlikely(err)) {
2108		if (!peeking) {
2109			udp_drops_inc(sk);
2110			UDP_INC_STATS(sock_net(sk),
2111				      UDP_MIB_INERRORS, is_udplite);
2112		}
2113		kfree_skb(skb);
2114		return err;
2115	}
2116
2117	if (!peeking)
2118		UDP_INC_STATS(sock_net(sk),
2119			      UDP_MIB_INDATAGRAMS, is_udplite);
2120
2121	sock_recv_cmsgs(msg, sk, skb);
2122
2123	/* Copy the address. */
2124	if (sin) {
2125		sin->sin_family = AF_INET;
2126		sin->sin_port = udp_hdr(skb)->source;
2127		sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
2128		memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
2129		*addr_len = sizeof(*sin);
2130
2131		BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
2132						      (struct sockaddr *)sin,
2133						      addr_len);
2134	}
2135
2136	if (udp_test_bit(GRO_ENABLED, sk))
2137		udp_cmsg_recv(msg, sk, skb);
2138
2139	if (inet_cmsg_flags(inet))
2140		ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
2141
2142	err = copied;
2143	if (flags & MSG_TRUNC)
2144		err = ulen;
2145
2146	skb_consume_udp(sk, skb, peeking ? -err : err);
2147	return err;
2148
2149csum_copy_err:
2150	if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
2151				 udp_skb_destructor)) {
2152		UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
2153		UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2154	}
2155	kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
2156
2157	/* starting over for a new packet, but check if we need to yield */
2158	cond_resched();
2159	msg->msg_flags &= ~MSG_TRUNC;
2160	goto try_again;
2161}
2162
2163int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
2164		    int addr_len)
2165{
2166	/* This check is replicated from __ip4_datagram_connect() and
2167	 * intended to prevent BPF program called below from accessing bytes
2168	 * that are out of the bound specified by user in addr_len.
2169	 */
2170	if (addr_len < sizeof(struct sockaddr_in))
2171		return -EINVAL;
2172
2173	return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, &addr_len);
2174}
2175EXPORT_IPV6_MOD(udp_pre_connect);
2176
2177static int udp_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
2178		       int addr_len)
2179{
2180	int res;
2181
2182	lock_sock(sk);
2183	res = __ip4_datagram_connect(sk, uaddr, addr_len);
2184	if (!res)
2185		udp4_hash4(sk);
2186	release_sock(sk);
2187	return res;
2188}
2189
2190int __udp_disconnect(struct sock *sk, int flags)
2191{
2192	struct inet_sock *inet = inet_sk(sk);
2193	/*
2194	 *	1003.1g - break association.
2195	 */
2196
2197	sk->sk_state = TCP_CLOSE;
2198	inet->inet_daddr = 0;
2199	inet->inet_dport = 0;
2200	sock_rps_reset_rxhash(sk);
2201	sk->sk_bound_dev_if = 0;
2202	if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
2203		inet_reset_saddr(sk);
2204		if (sk->sk_prot->rehash &&
2205		    (sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2206			sk->sk_prot->rehash(sk);
2207	}
2208
2209	if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
2210		sk->sk_prot->unhash(sk);
2211		inet->inet_sport = 0;
2212	}
2213	sk_dst_reset(sk);
2214	return 0;
2215}
2216EXPORT_SYMBOL(__udp_disconnect);
2217
2218int udp_disconnect(struct sock *sk, int flags)
2219{
2220	lock_sock(sk);
2221	__udp_disconnect(sk, flags);
2222	release_sock(sk);
2223	return 0;
2224}
2225EXPORT_IPV6_MOD(udp_disconnect);
2226
2227void udp_lib_unhash(struct sock *sk)
2228{
2229	if (sk_hashed(sk)) {
2230		struct udp_table *udptable = udp_get_table_prot(sk);
2231		struct udp_hslot *hslot, *hslot2;
2232
2233		sock_rps_delete_flow(sk);
2234		hslot  = udp_hashslot(udptable, sock_net(sk),
2235				      udp_sk(sk)->udp_port_hash);
2236		hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
2237
2238		spin_lock_bh(&hslot->lock);
2239		if (rcu_access_pointer(sk->sk_reuseport_cb))
2240			reuseport_detach_sock(sk);
2241		if (sk_del_node_init_rcu(sk)) {
2242			hslot->count--;
2243			inet_sk(sk)->inet_num = 0;
2244			sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
2245
2246			spin_lock(&hslot2->lock);
2247			hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
2248			hslot2->count--;
2249			spin_unlock(&hslot2->lock);
2250
2251			udp_unhash4(udptable, sk);
2252		}
2253		spin_unlock_bh(&hslot->lock);
2254	}
2255}
2256EXPORT_IPV6_MOD(udp_lib_unhash);
2257
2258/*
2259 * inet_rcv_saddr was changed, we must rehash secondary hash
2260 */
2261void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4)
2262{
2263	if (sk_hashed(sk)) {
2264		struct udp_table *udptable = udp_get_table_prot(sk);
2265		struct udp_hslot *hslot, *hslot2, *nhslot2;
2266
2267		hslot = udp_hashslot(udptable, sock_net(sk),
2268				     udp_sk(sk)->udp_port_hash);
2269		hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
2270		nhslot2 = udp_hashslot2(udptable, newhash);
2271
2272		if (hslot2 != nhslot2 ||
2273		    rcu_access_pointer(sk->sk_reuseport_cb)) {
2274			/* we must lock primary chain too */
2275			spin_lock_bh(&hslot->lock);
2276			if (rcu_access_pointer(sk->sk_reuseport_cb))
2277				reuseport_detach_sock(sk);
2278
2279			if (hslot2 != nhslot2) {
2280				spin_lock(&hslot2->lock);
2281				hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
2282				hslot2->count--;
2283				spin_unlock(&hslot2->lock);
2284
2285				spin_lock(&nhslot2->lock);
2286				hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
2287							 &nhslot2->head);
2288				nhslot2->count++;
2289				spin_unlock(&nhslot2->lock);
2290			}
2291
2292			spin_unlock_bh(&hslot->lock);
2293		}
2294
2295		/* Now process hash4 if necessary:
2296		 * (1) update hslot4;
2297		 * (2) update hslot2->hash4_cnt.
2298		 * Note that hslot2/hslot4 should be checked separately, as
2299		 * either of them may change with the other unchanged.
2300		 */
2301		if (udp_hashed4(sk)) {
2302			spin_lock_bh(&hslot->lock);
2303
2304			if (inet_rcv_saddr_any(sk)) {
2305				udp_unhash4(udptable, sk);
2306			} else {
2307				udp_rehash4(udptable, sk, newhash4);
2308				if (hslot2 != nhslot2) {
2309					spin_lock(&hslot2->lock);
2310					udp_hash4_dec(hslot2);
2311					spin_unlock(&hslot2->lock);
2312
2313					spin_lock(&nhslot2->lock);
2314					udp_hash4_inc(nhslot2);
2315					spin_unlock(&nhslot2->lock);
2316				}
2317			}
2318
2319			spin_unlock_bh(&hslot->lock);
2320		}
2321
2322		udp_sk(sk)->udp_portaddr_hash = newhash;
2323	}
2324}
2325EXPORT_IPV6_MOD(udp_lib_rehash);
2326
2327void udp_v4_rehash(struct sock *sk)
2328{
2329	u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
2330					  inet_sk(sk)->inet_rcv_saddr,
2331					  inet_sk(sk)->inet_num);
2332	u16 new_hash4 = udp_ehashfn(sock_net(sk),
2333				    sk->sk_rcv_saddr, sk->sk_num,
2334				    sk->sk_daddr, sk->sk_dport);
2335
2336	udp_lib_rehash(sk, new_hash, new_hash4);
2337}
2338
2339static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2340{
2341	int rc;
2342
2343	if (inet_sk(sk)->inet_daddr) {
2344		sock_rps_save_rxhash(sk, skb);
2345		sk_mark_napi_id(sk, skb);
2346		sk_incoming_cpu_update(sk);
2347	} else {
2348		sk_mark_napi_id_once(sk, skb);
2349	}
2350
2351	rc = __udp_enqueue_schedule_skb(sk, skb);
2352	if (rc < 0) {
2353		int is_udplite = IS_UDPLITE(sk);
2354		int drop_reason;
2355
2356		/* Note that an ENOMEM error is charged twice */
2357		if (rc == -ENOMEM) {
2358			UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
2359					is_udplite);
2360			drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
2361		} else {
2362			UDP_INC_STATS(sock_net(sk), UDP_MIB_MEMERRORS,
2363				      is_udplite);
2364			drop_reason = SKB_DROP_REASON_PROTO_MEM;
2365		}
2366		UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2367		trace_udp_fail_queue_rcv_skb(rc, sk, skb);
2368		sk_skb_reason_drop(sk, skb, drop_reason);
2369		return -1;
2370	}
2371
2372	return 0;
2373}
2374
2375/* returns:
2376 *  -1: error
2377 *   0: success
2378 *  >0: "udp encap" protocol resubmission
2379 *
2380 * Note that in the success and error cases, the skb is assumed to
2381 * have either been requeued or freed.
2382 */
2383static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
2384{
2385	enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2386	struct udp_sock *up = udp_sk(sk);
2387	int is_udplite = IS_UDPLITE(sk);
2388
2389	/*
2390	 *	Charge it to the socket, dropping if the queue is full.
2391	 */
2392	if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2393		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2394		goto drop;
2395	}
2396	nf_reset_ct(skb);
2397
2398	if (static_branch_unlikely(&udp_encap_needed_key) &&
2399	    READ_ONCE(up->encap_type)) {
2400		int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
2401
2402		/*
2403		 * This is an encapsulation socket so pass the skb to
2404		 * the socket's udp_encap_rcv() hook. Otherwise, just
2405		 * fall through and pass this up the UDP socket.
2406		 * up->encap_rcv() returns the following value:
2407		 * =0 if skb was successfully passed to the encap
2408		 *    handler or was discarded by it.
2409		 * >0 if skb should be passed on to UDP.
2410		 * <0 if skb should be resubmitted as proto -N
2411		 */
2412
2413		/* if we're overly short, let UDP handle it */
2414		encap_rcv = READ_ONCE(up->encap_rcv);
2415		if (encap_rcv) {
2416			int ret;
2417
2418			/* Verify checksum before giving to encap */
2419			if (udp_lib_checksum_complete(skb))
2420				goto csum_error;
2421
2422			ret = encap_rcv(sk, skb);
2423			if (ret <= 0) {
2424				__UDP_INC_STATS(sock_net(sk),
2425						UDP_MIB_INDATAGRAMS,
2426						is_udplite);
2427				return -ret;
2428			}
2429		}
2430
2431		/* FALLTHROUGH -- it's a UDP Packet */
2432	}
2433
2434	/*
2435	 * 	UDP-Lite specific tests, ignored on UDP sockets
2436	 */
2437	if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) {
2438		u16 pcrlen = READ_ONCE(up->pcrlen);
2439
2440		/*
2441		 * MIB statistics other than incrementing the error count are
2442		 * disabled for the following two types of errors: these depend
2443		 * on the application settings, not on the functioning of the
2444		 * protocol stack as such.
2445		 *
2446		 * RFC 3828 here recommends (sec 3.3): "There should also be a
2447		 * way ... to ... at least let the receiving application block
2448		 * delivery of packets with coverage values less than a value
2449		 * provided by the application."
2450		 */
2451		if (pcrlen == 0) {          /* full coverage was set  */
2452			net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2453					    UDP_SKB_CB(skb)->cscov, skb->len);
2454			goto drop;
2455		}
2456		/* The next case involves violating the min. coverage requested
2457		 * by the receiver. This is subtle: if receiver wants x and x is
2458		 * greater than the buffersize/MTU then receiver will complain
2459		 * that it wants x while sender emits packets of smaller size y.
2460		 * Therefore the above ...()->partial_cov statement is essential.
2461		 */
2462		if (UDP_SKB_CB(skb)->cscov < pcrlen) {
2463			net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2464					    UDP_SKB_CB(skb)->cscov, pcrlen);
2465			goto drop;
2466		}
2467	}
2468
2469	prefetch(&sk->sk_rmem_alloc);
2470	if (rcu_access_pointer(sk->sk_filter) &&
2471	    udp_lib_checksum_complete(skb))
2472			goto csum_error;
2473
2474	if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr), &drop_reason))
2475		goto drop;
2476
2477	udp_csum_pull_header(skb);
2478
2479	ipv4_pktinfo_prepare(sk, skb, true);
2480	return __udp_queue_rcv_skb(sk, skb);
2481
2482csum_error:
2483	drop_reason = SKB_DROP_REASON_UDP_CSUM;
2484	__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
2485drop:
2486	__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
2487	udp_drops_inc(sk);
2488	sk_skb_reason_drop(sk, skb, drop_reason);
2489	return -1;
2490}
2491
2492static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2493{
2494	struct sk_buff *next, *segs;
2495	int ret;
2496
2497	if (likely(!udp_unexpected_gso(sk, skb)))
2498		return udp_queue_rcv_one_skb(sk, skb);
2499
2500	BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
2501	__skb_push(skb, -skb_mac_offset(skb));
2502	segs = udp_rcv_segment(sk, skb, true);
2503	skb_list_walk_safe(segs, skb, next) {
2504		__skb_pull(skb, skb_transport_offset(skb));
2505
2506		udp_post_segment_fix_csum(skb);
2507		ret = udp_queue_rcv_one_skb(sk, skb);
2508		if (ret > 0)
2509			ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
2510	}
2511	return 0;
2512}
2513
2514/* For TCP sockets, sk_rx_dst is protected by socket lock
2515 * For UDP, we use xchg() to guard against concurrent changes.
2516 */
2517bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
2518{
2519	struct dst_entry *old;
2520
2521	if (dst_hold_safe(dst)) {
2522		old = unrcu_pointer(xchg(&sk->sk_rx_dst, RCU_INITIALIZER(dst)));
2523		dst_release(old);
2524		return old != dst;
2525	}
2526	return false;
2527}
2528EXPORT_IPV6_MOD(udp_sk_rx_dst_set);
2529
2530/*
2531 *	Multicasts and broadcasts go to each listener.
2532 *
2533 *	Note: called only from the BH handler context.
2534 */
2535static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
2536				    struct udphdr  *uh,
2537				    __be32 saddr, __be32 daddr,
2538				    struct udp_table *udptable,
2539				    int proto)
2540{
2541	struct sock *sk, *first = NULL;
2542	unsigned short hnum = ntohs(uh->dest);
2543	struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
2544	unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
2545	unsigned int offset = offsetof(typeof(*sk), sk_node);
2546	int dif = skb->dev->ifindex;
2547	int sdif = inet_sdif(skb);
2548	struct hlist_node *node;
2549	struct sk_buff *nskb;
2550
2551	if (use_hash2) {
2552		hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
2553			    udptable->mask;
2554		hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
2555start_lookup:
2556		hslot = &udptable->hash2[hash2].hslot;
2557		offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
2558	}
2559
2560	sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
2561		if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
2562					 uh->source, saddr, dif, sdif, hnum))
2563			continue;
2564
2565		if (!first) {
2566			first = sk;
2567			continue;
2568		}
2569		nskb = skb_clone(skb, GFP_ATOMIC);
2570
2571		if (unlikely(!nskb)) {
2572			udp_drops_inc(sk);
2573			__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
2574					IS_UDPLITE(sk));
2575			__UDP_INC_STATS(net, UDP_MIB_INERRORS,
2576					IS_UDPLITE(sk));
2577			continue;
2578		}
2579		if (udp_queue_rcv_skb(sk, nskb) > 0)
2580			consume_skb(nskb);
2581	}
2582
2583	/* Also lookup *:port if we are using hash2 and haven't done so yet. */
2584	if (use_hash2 && hash2 != hash2_any) {
2585		hash2 = hash2_any;
2586		goto start_lookup;
2587	}
2588
2589	if (first) {
2590		if (udp_queue_rcv_skb(first, skb) > 0)
2591			consume_skb(skb);
2592	} else {
2593		kfree_skb(skb);
2594		__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
2595				proto == IPPROTO_UDPLITE);
2596	}
2597	return 0;
2598}
2599
2600/* Initialize UDP checksum. If exited with zero value (success),
2601 * CHECKSUM_UNNECESSARY means, that no more checks are required.
2602 * Otherwise, csum completion requires checksumming packet body,
2603 * including udp header and folding it to skb->csum.
2604 */
2605static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
2606				 int proto)
2607{
2608	int err;
2609
2610	UDP_SKB_CB(skb)->partial_cov = 0;
2611	UDP_SKB_CB(skb)->cscov = skb->len;
2612
2613	if (proto == IPPROTO_UDPLITE) {
2614		err = udplite_checksum_init(skb, uh);
2615		if (err)
2616			return err;
2617
2618		if (UDP_SKB_CB(skb)->partial_cov) {
2619			skb->csum = inet_compute_pseudo(skb, proto);
2620			return 0;
2621		}
2622	}
2623
2624	/* Note, we are only interested in != 0 or == 0, thus the
2625	 * force to int.
2626	 */
2627	err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
2628							inet_compute_pseudo);
2629	if (err)
2630		return err;
2631
2632	if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
2633		/* If SW calculated the value, we know it's bad */
2634		if (skb->csum_complete_sw)
2635			return 1;
2636
2637		/* HW says the value is bad. Let's validate that.
2638		 * skb->csum is no longer the full packet checksum,
2639		 * so don't treat it as such.
2640		 */
2641		skb_checksum_complete_unset(skb);
2642	}
2643
2644	return 0;
2645}
2646
2647/* wrapper for udp_queue_rcv_skb taking care of csum conversion and
2648 * return code conversion for ip layer consumption
2649 */
2650static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
2651			       struct udphdr *uh)
2652{
2653	int ret;
2654
2655	if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
2656		skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
2657
2658	ret = udp_queue_rcv_skb(sk, skb);
2659
2660	/* a return value > 0 means to resubmit the input, but
2661	 * it wants the return to be -protocol, or 0
2662	 */
2663	if (ret > 0)
2664		return -ret;
2665	return 0;
2666}
2667
2668/*
2669 *	All we need to do is get the socket, and then do a checksum.
2670 */
2671
2672int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
2673		   int proto)
2674{
2675	struct sock *sk = NULL;
2676	struct udphdr *uh;
2677	unsigned short ulen;
2678	struct rtable *rt = skb_rtable(skb);
2679	__be32 saddr, daddr;
2680	struct net *net = dev_net(skb->dev);
2681	bool refcounted;
2682	int drop_reason;
2683
2684	drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
2685
2686	/*
2687	 *  Validate the packet.
2688	 */
2689	if (!pskb_may_pull(skb, sizeof(struct udphdr)))
2690		goto drop;		/* No space for header. */
2691
2692	uh   = udp_hdr(skb);
2693	ulen = ntohs(uh->len);
2694	saddr = ip_hdr(skb)->saddr;
2695	daddr = ip_hdr(skb)->daddr;
2696
2697	if (ulen > skb->len)
2698		goto short_packet;
2699
2700	if (proto == IPPROTO_UDP) {
2701		/* UDP validates ulen. */
2702		if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
2703			goto short_packet;
2704		uh = udp_hdr(skb);
2705	}
2706
2707	if (udp4_csum_init(skb, uh, proto))
2708		goto csum_error;
2709
2710	sk = inet_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
2711			     &refcounted, udp_ehashfn);
2712	if (IS_ERR(sk))
2713		goto no_sk;
2714
2715	if (sk) {
2716		struct dst_entry *dst = skb_dst(skb);
2717		int ret;
2718
2719		if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
2720			udp_sk_rx_dst_set(sk, dst);
2721
2722		ret = udp_unicast_rcv_skb(sk, skb, uh);
2723		if (refcounted)
2724			sock_put(sk);
2725		return ret;
2726	}
2727
2728	if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
2729		return __udp4_lib_mcast_deliver(net, skb, uh,
2730						saddr, daddr, udptable, proto);
2731
2732	sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
2733	if (sk)
2734		return udp_unicast_rcv_skb(sk, skb, uh);
2735no_sk:
2736	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2737		goto drop;
2738	nf_reset_ct(skb);
2739
2740	/* No socket. Drop packet silently, if checksum is wrong */
2741	if (udp_lib_checksum_complete(skb))
2742		goto csum_error;
2743
2744	drop_reason = SKB_DROP_REASON_NO_SOCKET;
2745	__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
2746	icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
2747
2748	/*
2749	 * Hmm.  We got an UDP packet to a port to which we
2750	 * don't wanna listen.  Ignore it.
2751	 */
2752	sk_skb_reason_drop(sk, skb, drop_reason);
2753	return 0;
2754
2755short_packet:
2756	drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
2757	net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
2758			    proto == IPPROTO_UDPLITE ? "Lite" : "",
2759			    &saddr, ntohs(uh->source),
2760			    ulen, skb->len,
2761			    &daddr, ntohs(uh->dest));
2762	goto drop;
2763
2764csum_error:
2765	/*
2766	 * RFC1122: OK.  Discards the bad packet silently (as far as
2767	 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
2768	 */
2769	drop_reason = SKB_DROP_REASON_UDP_CSUM;
2770	net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
2771			    proto == IPPROTO_UDPLITE ? "Lite" : "",
2772			    &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
2773			    ulen);
2774	__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
2775drop:
2776	__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
2777	sk_skb_reason_drop(sk, skb, drop_reason);
2778	return 0;
2779}
2780
2781/* We can only early demux multicast if there is a single matching socket.
2782 * If more than one socket found returns NULL
2783 */
2784static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
2785						  __be16 loc_port, __be32 loc_addr,
2786						  __be16 rmt_port, __be32 rmt_addr,
2787						  int dif, int sdif)
2788{
2789	struct udp_table *udptable = net->ipv4.udp_table;
2790	unsigned short hnum = ntohs(loc_port);
2791	struct sock *sk, *result;
2792	struct udp_hslot *hslot;
2793	unsigned int slot;
2794
2795	slot = udp_hashfn(net, hnum, udptable->mask);
2796	hslot = &udptable->hash[slot];
2797
2798	/* Do not bother scanning a too big list */
2799	if (hslot->count > 10)
2800		return NULL;
2801
2802	result = NULL;
2803	sk_for_each_rcu(sk, &hslot->head) {
2804		if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
2805					rmt_port, rmt_addr, dif, sdif, hnum)) {
2806			if (result)
2807				return NULL;
2808			result = sk;
2809		}
2810	}
2811
2812	return result;
2813}
2814
2815/* For unicast we should only early demux connected sockets or we can
2816 * break forwarding setups.  The chains here can be long so only check
2817 * if the first socket is an exact match and if not move on.
2818 */
2819static struct sock *__udp4_lib_demux_lookup(struct net *net,
2820					    __be16 loc_port, __be32 loc_addr,
2821					    __be16 rmt_port, __be32 rmt_addr,
2822					    int dif, int sdif)
2823{
2824	struct udp_table *udptable = net->ipv4.udp_table;
2825	INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
2826	unsigned short hnum = ntohs(loc_port);
2827	struct udp_hslot *hslot2;
2828	unsigned int hash2;
2829	__portpair ports;
2830	struct sock *sk;
2831
2832	hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
2833	hslot2 = udp_hashslot2(udptable, hash2);
2834	ports = INET_COMBINED_PORTS(rmt_port, hnum);
2835
2836	udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2837		if (inet_match(net, sk, acookie, ports, dif, sdif))
2838			return sk;
2839		/* Only check first socket in chain */
2840		break;
2841	}
2842	return NULL;
2843}
2844
2845enum skb_drop_reason udp_v4_early_demux(struct sk_buff *skb)
2846{
2847	struct net *net = dev_net(skb->dev);
2848	struct in_device *in_dev = NULL;
2849	const struct iphdr *iph;
2850	const struct udphdr *uh;
2851	struct sock *sk = NULL;
2852	struct dst_entry *dst;
2853	int dif = skb->dev->ifindex;
2854	int sdif = inet_sdif(skb);
2855	int ours;
2856
2857	/* validate the packet */
2858	if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
2859		return SKB_NOT_DROPPED_YET;
2860
2861	iph = ip_hdr(skb);
2862	uh = udp_hdr(skb);
2863
2864	if (skb->pkt_type == PACKET_MULTICAST) {
2865		in_dev = __in_dev_get_rcu(skb->dev);
2866
2867		if (!in_dev)
2868			return SKB_NOT_DROPPED_YET;
2869
2870		ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
2871				       iph->protocol);
2872		if (!ours)
2873			return SKB_NOT_DROPPED_YET;
2874
2875		sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
2876						   uh->source, iph->saddr,
2877						   dif, sdif);
2878	} else if (skb->pkt_type == PACKET_HOST) {
2879		sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
2880					     uh->source, iph->saddr, dif, sdif);
2881	}
2882
2883	if (!sk)
2884		return SKB_NOT_DROPPED_YET;
2885
2886	skb->sk = sk;
2887	DEBUG_NET_WARN_ON_ONCE(sk_is_refcounted(sk));
2888	skb->destructor = sock_pfree;
2889	dst = rcu_dereference(sk->sk_rx_dst);
2890
2891	if (dst)
2892		dst = dst_check(dst, 0);
2893	if (dst) {
2894		u32 itag = 0;
2895
2896		/* set noref for now.
2897		 * any place which wants to hold dst has to call
2898		 * dst_hold_safe()
2899		 */
2900		skb_dst_set_noref(skb, dst);
2901
2902		/* for unconnected multicast sockets we need to validate
2903		 * the source on each packet
2904		 */
2905		if (!inet_sk(sk)->inet_daddr && in_dev)
2906			return ip_mc_validate_source(skb, iph->daddr,
2907						     iph->saddr,
2908						     ip4h_dscp(iph),
2909						     skb->dev, in_dev, &itag);
2910	}
2911	return SKB_NOT_DROPPED_YET;
2912}
2913
2914int udp_rcv(struct sk_buff *skb)
2915{
2916	return __udp4_lib_rcv(skb, dev_net(skb->dev)->ipv4.udp_table, IPPROTO_UDP);
2917}
2918
2919void udp_destroy_sock(struct sock *sk)
2920{
2921	struct udp_sock *up = udp_sk(sk);
2922	bool slow = lock_sock_fast(sk);
2923
2924	/* protects from races with udp_abort() */
2925	sock_set_flag(sk, SOCK_DEAD);
2926	udp_flush_pending_frames(sk);
2927	unlock_sock_fast(sk, slow);
2928	if (static_branch_unlikely(&udp_encap_needed_key)) {
2929		if (up->encap_type) {
2930			void (*encap_destroy)(struct sock *sk);
2931			encap_destroy = READ_ONCE(up->encap_destroy);
2932			if (encap_destroy)
2933				encap_destroy(sk);
2934		}
2935		if (udp_test_bit(ENCAP_ENABLED, sk)) {
2936			static_branch_dec(&udp_encap_needed_key);
2937			udp_tunnel_cleanup_gro(sk);
2938		}
2939	}
2940}
2941
2942typedef struct sk_buff *(*udp_gro_receive_t)(struct sock *sk,
2943					     struct list_head *head,
2944					     struct sk_buff *skb);
2945
2946static void set_xfrm_gro_udp_encap_rcv(__u16 encap_type, unsigned short family,
2947				       struct sock *sk)
2948{
2949#ifdef CONFIG_XFRM
2950	udp_gro_receive_t new_gro_receive;
2951
2952	if (udp_test_bit(GRO_ENABLED, sk) && encap_type == UDP_ENCAP_ESPINUDP) {
2953		if (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6)
2954			new_gro_receive = ipv6_stub->xfrm6_gro_udp_encap_rcv;
2955		else
2956			new_gro_receive = xfrm4_gro_udp_encap_rcv;
2957
2958		if (udp_sk(sk)->gro_receive != new_gro_receive) {
2959			/*
2960			 * With IPV6_ADDRFORM the gro callback could change
2961			 * after being set, unregister the old one, if valid.
2962			 */
2963			if (udp_sk(sk)->gro_receive)
2964				udp_tunnel_update_gro_rcv(sk, false);
2965
2966			WRITE_ONCE(udp_sk(sk)->gro_receive, new_gro_receive);
2967			udp_tunnel_update_gro_rcv(sk, true);
2968		}
2969	}
2970#endif
2971}
2972
2973/*
2974 *	Socket option code for UDP
2975 */
2976int udp_lib_setsockopt(struct sock *sk, int level, int optname,
2977		       sockptr_t optval, unsigned int optlen,
2978		       int (*push_pending_frames)(struct sock *))
2979{
2980	struct udp_sock *up = udp_sk(sk);
2981	int val, valbool;
2982	int err = 0;
2983	int is_udplite = IS_UDPLITE(sk);
2984
2985	if (level == SOL_SOCKET) {
2986		err = sk_setsockopt(sk, level, optname, optval, optlen);
2987
2988		if (optname == SO_RCVBUF || optname == SO_RCVBUFFORCE) {
2989			sockopt_lock_sock(sk);
2990			/* paired with READ_ONCE in udp_rmem_release() */
2991			WRITE_ONCE(up->forward_threshold, sk->sk_rcvbuf >> 2);
2992			sockopt_release_sock(sk);
2993		}
2994		return err;
2995	}
2996
2997	if (optlen < sizeof(int))
2998		return -EINVAL;
2999
3000	if (copy_from_sockptr(&val, optval, sizeof(val)))
3001		return -EFAULT;
3002
3003	valbool = val ? 1 : 0;
3004
3005	switch (optname) {
3006	case UDP_CORK:
3007		if (val != 0) {
3008			udp_set_bit(CORK, sk);
3009		} else {
3010			udp_clear_bit(CORK, sk);
3011			lock_sock(sk);
3012			push_pending_frames(sk);
3013			release_sock(sk);
3014		}
3015		break;
3016
3017	case UDP_ENCAP:
3018		sockopt_lock_sock(sk);
3019		switch (val) {
3020		case 0:
3021#ifdef CONFIG_XFRM
3022		case UDP_ENCAP_ESPINUDP:
3023			set_xfrm_gro_udp_encap_rcv(val, sk->sk_family, sk);
3024#if IS_ENABLED(CONFIG_IPV6)
3025			if (sk->sk_family == AF_INET6)
3026				WRITE_ONCE(up->encap_rcv,
3027					   ipv6_stub->xfrm6_udp_encap_rcv);
3028			else
3029#endif
3030				WRITE_ONCE(up->encap_rcv,
3031					   xfrm4_udp_encap_rcv);
3032#endif
3033			fallthrough;
3034		case UDP_ENCAP_L2TPINUDP:
3035			WRITE_ONCE(up->encap_type, val);
3036			udp_tunnel_encap_enable(sk);
3037			break;
3038		default:
3039			err = -ENOPROTOOPT;
3040			break;
3041		}
3042		sockopt_release_sock(sk);
3043		break;
3044
3045	case UDP_NO_CHECK6_TX:
3046		udp_set_no_check6_tx(sk, valbool);
3047		break;
3048
3049	case UDP_NO_CHECK6_RX:
3050		udp_set_no_check6_rx(sk, valbool);
3051		break;
3052
3053	case UDP_SEGMENT:
3054		if (val < 0 || val > USHRT_MAX)
3055			return -EINVAL;
3056		WRITE_ONCE(up->gso_size, val);
3057		break;
3058
3059	case UDP_GRO:
3060		sockopt_lock_sock(sk);
3061		/* when enabling GRO, accept the related GSO packet type */
3062		if (valbool)
3063			udp_tunnel_encap_enable(sk);
3064		udp_assign_bit(GRO_ENABLED, sk, valbool);
3065		udp_assign_bit(ACCEPT_L4, sk, valbool);
3066		set_xfrm_gro_udp_encap_rcv(up->encap_type, sk->sk_family, sk);
3067		sockopt_release_sock(sk);
3068		break;
3069
3070	/*
3071	 * 	UDP-Lite's partial checksum coverage (RFC 3828).
3072	 */
3073	/* The sender sets actual checksum coverage length via this option.
3074	 * The case coverage > packet length is handled by send module. */
3075	case UDPLITE_SEND_CSCOV:
3076		if (!is_udplite)         /* Disable the option on UDP sockets */
3077			return -ENOPROTOOPT;
3078		if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
3079			val = 8;
3080		else if (val > USHRT_MAX)
3081			val = USHRT_MAX;
3082		WRITE_ONCE(up->pcslen, val);
3083		udp_set_bit(UDPLITE_SEND_CC, sk);
3084		break;
3085
3086	/* The receiver specifies a minimum checksum coverage value. To make
3087	 * sense, this should be set to at least 8 (as done below). If zero is
3088	 * used, this again means full checksum coverage.                     */
3089	case UDPLITE_RECV_CSCOV:
3090		if (!is_udplite)         /* Disable the option on UDP sockets */
3091			return -ENOPROTOOPT;
3092		if (val != 0 && val < 8) /* Avoid silly minimal values.       */
3093			val = 8;
3094		else if (val > USHRT_MAX)
3095			val = USHRT_MAX;
3096		WRITE_ONCE(up->pcrlen, val);
3097		udp_set_bit(UDPLITE_RECV_CC, sk);
3098		break;
3099
3100	default:
3101		err = -ENOPROTOOPT;
3102		break;
3103	}
3104
3105	return err;
3106}
3107EXPORT_IPV6_MOD(udp_lib_setsockopt);
3108
3109int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
3110		   unsigned int optlen)
3111{
3112	if (level == SOL_UDP  ||  level == SOL_UDPLITE || level == SOL_SOCKET)
3113		return udp_lib_setsockopt(sk, level, optname,
3114					  optval, optlen,
3115					  udp_push_pending_frames);
3116	return ip_setsockopt(sk, level, optname, optval, optlen);
3117}
3118
3119int udp_lib_getsockopt(struct sock *sk, int level, int optname,
3120		       char __user *optval, int __user *optlen)
3121{
3122	struct udp_sock *up = udp_sk(sk);
3123	int val, len;
3124
3125	if (get_user(len, optlen))
3126		return -EFAULT;
3127
3128	if (len < 0)
3129		return -EINVAL;
3130
3131	len = min_t(unsigned int, len, sizeof(int));
3132
3133	switch (optname) {
3134	case UDP_CORK:
3135		val = udp_test_bit(CORK, sk);
3136		break;
3137
3138	case UDP_ENCAP:
3139		val = READ_ONCE(up->encap_type);
3140		break;
3141
3142	case UDP_NO_CHECK6_TX:
3143		val = udp_get_no_check6_tx(sk);
3144		break;
3145
3146	case UDP_NO_CHECK6_RX:
3147		val = udp_get_no_check6_rx(sk);
3148		break;
3149
3150	case UDP_SEGMENT:
3151		val = READ_ONCE(up->gso_size);
3152		break;
3153
3154	case UDP_GRO:
3155		val = udp_test_bit(GRO_ENABLED, sk);
3156		break;
3157
3158	/* The following two cannot be changed on UDP sockets, the return is
3159	 * always 0 (which corresponds to the full checksum coverage of UDP). */
3160	case UDPLITE_SEND_CSCOV:
3161		val = READ_ONCE(up->pcslen);
3162		break;
3163
3164	case UDPLITE_RECV_CSCOV:
3165		val = READ_ONCE(up->pcrlen);
3166		break;
3167
3168	default:
3169		return -ENOPROTOOPT;
3170	}
3171
3172	if (put_user(len, optlen))
3173		return -EFAULT;
3174	if (copy_to_user(optval, &val, len))
3175		return -EFAULT;
3176	return 0;
3177}
3178EXPORT_IPV6_MOD(udp_lib_getsockopt);
3179
3180int udp_getsockopt(struct sock *sk, int level, int optname,
3181		   char __user *optval, int __user *optlen)
3182{
3183	if (level == SOL_UDP  ||  level == SOL_UDPLITE)
3184		return udp_lib_getsockopt(sk, level, optname, optval, optlen);
3185	return ip_getsockopt(sk, level, optname, optval, optlen);
3186}
3187
3188/**
3189 * 	udp_poll - wait for a UDP event.
3190 *	@file: - file struct
3191 *	@sock: - socket
3192 *	@wait: - poll table
3193 *
3194 *	This is same as datagram poll, except for the special case of
3195 *	blocking sockets. If application is using a blocking fd
3196 *	and a packet with checksum error is in the queue;
3197 *	then it could get return from select indicating data available
3198 *	but then block when reading it. Add special case code
3199 *	to work around these arguably broken applications.
3200 */
3201__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
3202{
3203	__poll_t mask = datagram_poll(file, sock, wait);
3204	struct sock *sk = sock->sk;
3205
3206	if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
3207		mask |= EPOLLIN | EPOLLRDNORM;
3208
3209	/* Check for false positives due to checksum errors */
3210	if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
3211	    !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
3212		mask &= ~(EPOLLIN | EPOLLRDNORM);
3213
3214	/* psock ingress_msg queue should not contain any bad checksum frames */
3215	if (sk_is_readable(sk))
3216		mask |= EPOLLIN | EPOLLRDNORM;
3217	return mask;
3218
3219}
3220EXPORT_IPV6_MOD(udp_poll);
3221
3222int udp_abort(struct sock *sk, int err)
3223{
3224	if (!has_current_bpf_ctx())
3225		lock_sock(sk);
3226
3227	/* udp{v6}_destroy_sock() sets it under the sk lock, avoid racing
3228	 * with close()
3229	 */
3230	if (sock_flag(sk, SOCK_DEAD))
3231		goto out;
3232
3233	sk->sk_err = err;
3234	sk_error_report(sk);
3235	__udp_disconnect(sk, 0);
3236
3237out:
3238	if (!has_current_bpf_ctx())
3239		release_sock(sk);
3240
3241	return 0;
3242}
3243EXPORT_IPV6_MOD_GPL(udp_abort);
3244
3245struct proto udp_prot = {
3246	.name			= "UDP",
3247	.owner			= THIS_MODULE,
3248	.close			= udp_lib_close,
3249	.pre_connect		= udp_pre_connect,
3250	.connect		= udp_connect,
3251	.disconnect		= udp_disconnect,
3252	.ioctl			= udp_ioctl,
3253	.init			= udp_init_sock,
3254	.destroy		= udp_destroy_sock,
3255	.setsockopt		= udp_setsockopt,
3256	.getsockopt		= udp_getsockopt,
3257	.sendmsg		= udp_sendmsg,
3258	.recvmsg		= udp_recvmsg,
3259	.splice_eof		= udp_splice_eof,
3260	.release_cb		= ip4_datagram_release_cb,
3261	.hash			= udp_lib_hash,
3262	.unhash			= udp_lib_unhash,
3263	.rehash			= udp_v4_rehash,
3264	.get_port		= udp_v4_get_port,
3265	.put_port		= udp_lib_unhash,
3266#ifdef CONFIG_BPF_SYSCALL
3267	.psock_update_sk_prot	= udp_bpf_update_proto,
3268#endif
3269	.memory_allocated	= &net_aligned_data.udp_memory_allocated,
3270	.per_cpu_fw_alloc	= &udp_memory_per_cpu_fw_alloc,
3271
3272	.sysctl_mem		= sysctl_udp_mem,
3273	.sysctl_wmem_offset	= offsetof(struct net, ipv4.sysctl_udp_wmem_min),
3274	.sysctl_rmem_offset	= offsetof(struct net, ipv4.sysctl_udp_rmem_min),
3275	.obj_size		= sizeof(struct udp_sock),
3276	.h.udp_table		= NULL,
3277	.diag_destroy		= udp_abort,
3278};
3279EXPORT_SYMBOL(udp_prot);
3280
3281/* ------------------------------------------------------------------------ */
3282#ifdef CONFIG_PROC_FS
3283
3284static unsigned short seq_file_family(const struct seq_file *seq);
3285static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
3286{
3287	unsigned short family = seq_file_family(seq);
3288
3289	/* AF_UNSPEC is used as a match all */
3290	return ((family == AF_UNSPEC || family == sk->sk_family) &&
3291		net_eq(sock_net(sk), seq_file_net(seq)));
3292}
3293
3294#ifdef CONFIG_BPF_SYSCALL
3295static const struct seq_operations bpf_iter_udp_seq_ops;
3296#endif
3297static struct udp_table *udp_get_table_seq(struct seq_file *seq,
3298					   struct net *net)
3299{
3300	const struct udp_seq_afinfo *afinfo;
3301
3302#ifdef CONFIG_BPF_SYSCALL
3303	if (seq->op == &bpf_iter_udp_seq_ops)
3304		return net->ipv4.udp_table;
3305#endif
3306
3307	afinfo = pde_data(file_inode(seq->file));
3308	return afinfo->udp_table ? : net->ipv4.udp_table;
3309}
3310
3311static struct sock *udp_get_first(struct seq_file *seq, int start)
3312{
3313	struct udp_iter_state *state = seq->private;
3314	struct net *net = seq_file_net(seq);
3315	struct udp_table *udptable;
3316	struct sock *sk;
3317
3318	udptable = udp_get_table_seq(seq, net);
3319
3320	for (state->bucket = start; state->bucket <= udptable->mask;
3321	     ++state->bucket) {
3322		struct udp_hslot *hslot = &udptable->hash[state->bucket];
3323
3324		if (hlist_empty(&hslot->head))
3325			continue;
3326
3327		spin_lock_bh(&hslot->lock);
3328		sk_for_each(sk, &hslot->head) {
3329			if (seq_sk_match(seq, sk))
3330				goto found;
3331		}
3332		spin_unlock_bh(&hslot->lock);
3333	}
3334	sk = NULL;
3335found:
3336	return sk;
3337}
3338
3339static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
3340{
3341	struct udp_iter_state *state = seq->private;
3342	struct net *net = seq_file_net(seq);
3343	struct udp_table *udptable;
3344
3345	do {
3346		sk = sk_next(sk);
3347	} while (sk && !seq_sk_match(seq, sk));
3348
3349	if (!sk) {
3350		udptable = udp_get_table_seq(seq, net);
3351
3352		if (state->bucket <= udptable->mask)
3353			spin_unlock_bh(&udptable->hash[state->bucket].lock);
3354
3355		return udp_get_first(seq, state->bucket + 1);
3356	}
3357	return sk;
3358}
3359
3360static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
3361{
3362	struct sock *sk = udp_get_first(seq, 0);
3363
3364	if (sk)
3365		while (pos && (sk = udp_get_next(seq, sk)) != NULL)
3366			--pos;
3367	return pos ? NULL : sk;
3368}
3369
3370void *udp_seq_start(struct seq_file *seq, loff_t *pos)
3371{
3372	struct udp_iter_state *state = seq->private;
3373	state->bucket = MAX_UDP_PORTS;
3374
3375	return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
3376}
3377EXPORT_IPV6_MOD(udp_seq_start);
3378
3379void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3380{
3381	struct sock *sk;
3382
3383	if (v == SEQ_START_TOKEN)
3384		sk = udp_get_idx(seq, 0);
3385	else
3386		sk = udp_get_next(seq, v);
3387
3388	++*pos;
3389	return sk;
3390}
3391EXPORT_IPV6_MOD(udp_seq_next);
3392
3393void udp_seq_stop(struct seq_file *seq, void *v)
3394{
3395	struct udp_iter_state *state = seq->private;
3396	struct udp_table *udptable;
3397
3398	udptable = udp_get_table_seq(seq, seq_file_net(seq));
3399
3400	if (state->bucket <= udptable->mask)
3401		spin_unlock_bh(&udptable->hash[state->bucket].lock);
3402}
3403EXPORT_IPV6_MOD(udp_seq_stop);
3404
3405/* ------------------------------------------------------------------------ */
3406static void udp4_format_sock(struct sock *sp, struct seq_file *f,
3407		int bucket)
3408{
3409	struct inet_sock *inet = inet_sk(sp);
3410	__be32 dest = inet->inet_daddr;
3411	__be32 src  = inet->inet_rcv_saddr;
3412	__u16 destp	  = ntohs(inet->inet_dport);
3413	__u16 srcp	  = ntohs(inet->inet_sport);
3414
3415	seq_printf(f, "%5d: %08X:%04X %08X:%04X"
3416		" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
3417		bucket, src, srcp, dest, destp, sp->sk_state,
3418		sk_wmem_alloc_get(sp),
3419		udp_rqueue_get(sp),
3420		0, 0L, 0,
3421		from_kuid_munged(seq_user_ns(f), sk_uid(sp)),
3422		0, sock_i_ino(sp),
3423		refcount_read(&sp->sk_refcnt), sp,
3424		sk_drops_read(sp));
3425}
3426
3427int udp4_seq_show(struct seq_file *seq, void *v)
3428{
3429	seq_setwidth(seq, 127);
3430	if (v == SEQ_START_TOKEN)
3431		seq_puts(seq, "   sl  local_address rem_address   st tx_queue "
3432			   "rx_queue tr tm->when retrnsmt   uid  timeout "
3433			   "inode ref pointer drops");
3434	else {
3435		struct udp_iter_state *state = seq->private;
3436
3437		udp4_format_sock(v, seq, state->bucket);
3438	}
3439	seq_pad(seq, '\n');
3440	return 0;
3441}
3442
3443#ifdef CONFIG_BPF_SYSCALL
3444struct bpf_iter__udp {
3445	__bpf_md_ptr(struct bpf_iter_meta *, meta);
3446	__bpf_md_ptr(struct udp_sock *, udp_sk);
3447	uid_t uid __aligned(8);
3448	int bucket __aligned(8);
3449};
3450
3451union bpf_udp_iter_batch_item {
3452	struct sock *sk;
3453	__u64 cookie;
3454};
3455
3456struct bpf_udp_iter_state {
3457	struct udp_iter_state state;
3458	unsigned int cur_sk;
3459	unsigned int end_sk;
3460	unsigned int max_sk;
3461	union bpf_udp_iter_batch_item *batch;
3462};
3463
3464static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
3465				      unsigned int new_batch_sz, gfp_t flags);
3466static struct sock *bpf_iter_udp_resume(struct sock *first_sk,
3467					union bpf_udp_iter_batch_item *cookies,
3468					int n_cookies)
3469{
3470	struct sock *sk = NULL;
3471	int i;
3472
3473	for (i = 0; i < n_cookies; i++) {
3474		sk = first_sk;
3475		udp_portaddr_for_each_entry_from(sk)
3476			if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
3477				goto done;
3478	}
3479done:
3480	return sk;
3481}
3482
3483static struct sock *bpf_iter_udp_batch(struct seq_file *seq)
3484{
3485	struct bpf_udp_iter_state *iter = seq->private;
3486	struct udp_iter_state *state = &iter->state;
3487	unsigned int find_cookie, end_cookie;
3488	struct net *net = seq_file_net(seq);
3489	struct udp_table *udptable;
3490	unsigned int batch_sks = 0;
3491	int resume_bucket;
3492	int resizes = 0;
3493	struct sock *sk;
3494	int err = 0;
3495
3496	resume_bucket = state->bucket;
3497
3498	/* The current batch is done, so advance the bucket. */
3499	if (iter->cur_sk == iter->end_sk)
3500		state->bucket++;
3501
3502	udptable = udp_get_table_seq(seq, net);
3503
3504again:
3505	/* New batch for the next bucket.
3506	 * Iterate over the hash table to find a bucket with sockets matching
3507	 * the iterator attributes, and return the first matching socket from
3508	 * the bucket. The remaining matched sockets from the bucket are batched
3509	 * before releasing the bucket lock. This allows BPF programs that are
3510	 * called in seq_show to acquire the bucket lock if needed.
3511	 */
3512	find_cookie = iter->cur_sk;
3513	end_cookie = iter->end_sk;
3514	iter->cur_sk = 0;
3515	iter->end_sk = 0;
3516	batch_sks = 0;
3517
3518	for (; state->bucket <= udptable->mask; state->bucket++) {
3519		struct udp_hslot *hslot2 = &udptable->hash2[state->bucket].hslot;
3520
3521		if (hlist_empty(&hslot2->head))
3522			goto next_bucket;
3523
3524		spin_lock_bh(&hslot2->lock);
3525		sk = hlist_entry_safe(hslot2->head.first, struct sock,
3526				      __sk_common.skc_portaddr_node);
3527		/* Resume from the first (in iteration order) unseen socket from
3528		 * the last batch that still exists in resume_bucket. Most of
3529		 * the time this will just be where the last iteration left off
3530		 * in resume_bucket unless that socket disappeared between
3531		 * reads.
3532		 */
3533		if (state->bucket == resume_bucket)
3534			sk = bpf_iter_udp_resume(sk, &iter->batch[find_cookie],
3535						 end_cookie - find_cookie);
3536fill_batch:
3537		udp_portaddr_for_each_entry_from(sk) {
3538			if (seq_sk_match(seq, sk)) {
3539				if (iter->end_sk < iter->max_sk) {
3540					sock_hold(sk);
3541					iter->batch[iter->end_sk++].sk = sk;
3542				}
3543				batch_sks++;
3544			}
3545		}
3546
3547		/* Allocate a larger batch and try again. */
3548		if (unlikely(resizes <= 1 && iter->end_sk &&
3549			     iter->end_sk != batch_sks)) {
3550			resizes++;
3551
3552			/* First, try with GFP_USER to maximize the chances of
3553			 * grabbing more memory.
3554			 */
3555			if (resizes == 1) {
3556				spin_unlock_bh(&hslot2->lock);
3557				err = bpf_iter_udp_realloc_batch(iter,
3558								 batch_sks * 3 / 2,
3559								 GFP_USER);
3560				if (err)
3561					return ERR_PTR(err);
3562				/* Start over. */
3563				goto again;
3564			}
3565
3566			/* Next, hold onto the lock, so the bucket doesn't
3567			 * change while we get the rest of the sockets.
3568			 */
3569			err = bpf_iter_udp_realloc_batch(iter, batch_sks,
3570							 GFP_NOWAIT);
3571			if (err) {
3572				spin_unlock_bh(&hslot2->lock);
3573				return ERR_PTR(err);
3574			}
3575
3576			/* Pick up where we left off. */
3577			sk = iter->batch[iter->end_sk - 1].sk;
3578			sk = hlist_entry_safe(sk->__sk_common.skc_portaddr_node.next,
3579					      struct sock,
3580					      __sk_common.skc_portaddr_node);
3581			batch_sks = iter->end_sk;
3582			goto fill_batch;
3583		}
3584
3585		spin_unlock_bh(&hslot2->lock);
3586
3587		if (iter->end_sk)
3588			break;
3589next_bucket:
3590		resizes = 0;
3591	}
3592
3593	WARN_ON_ONCE(iter->end_sk != batch_sks);
3594	return iter->end_sk ? iter->batch[0].sk : NULL;
3595}
3596
3597static void *bpf_iter_udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3598{
3599	struct bpf_udp_iter_state *iter = seq->private;
3600	struct sock *sk;
3601
3602	/* Whenever seq_next() is called, the iter->cur_sk is
3603	 * done with seq_show(), so unref the iter->cur_sk.
3604	 */
3605	if (iter->cur_sk < iter->end_sk)
3606		sock_put(iter->batch[iter->cur_sk++].sk);
3607
3608	/* After updating iter->cur_sk, check if there are more sockets
3609	 * available in the current bucket batch.
3610	 */
3611	if (iter->cur_sk < iter->end_sk)
3612		sk = iter->batch[iter->cur_sk].sk;
3613	else
3614		/* Prepare a new batch. */
3615		sk = bpf_iter_udp_batch(seq);
3616
3617	++*pos;
3618	return sk;
3619}
3620
3621static void *bpf_iter_udp_seq_start(struct seq_file *seq, loff_t *pos)
3622{
3623	/* bpf iter does not support lseek, so it always
3624	 * continue from where it was stop()-ped.
3625	 */
3626	if (*pos)
3627		return bpf_iter_udp_batch(seq);
3628
3629	return SEQ_START_TOKEN;
3630}
3631
3632static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
3633			     struct udp_sock *udp_sk, uid_t uid, int bucket)
3634{
3635	struct bpf_iter__udp ctx;
3636
3637	meta->seq_num--;  /* skip SEQ_START_TOKEN */
3638	ctx.meta = meta;
3639	ctx.udp_sk = udp_sk;
3640	ctx.uid = uid;
3641	ctx.bucket = bucket;
3642	return bpf_iter_run_prog(prog, &ctx);
3643}
3644
3645static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v)
3646{
3647	struct udp_iter_state *state = seq->private;
3648	struct bpf_iter_meta meta;
3649	struct bpf_prog *prog;
3650	struct sock *sk = v;
3651	uid_t uid;
3652	int ret;
3653
3654	if (v == SEQ_START_TOKEN)
3655		return 0;
3656
3657	lock_sock(sk);
3658
3659	if (unlikely(sk_unhashed(sk))) {
3660		ret = SEQ_SKIP;
3661		goto unlock;
3662	}
3663
3664	uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
3665	meta.seq = seq;
3666	prog = bpf_iter_get_info(&meta, false);
3667	ret = udp_prog_seq_show(prog, &meta, v, uid, state->bucket);
3668
3669unlock:
3670	release_sock(sk);
3671	return ret;
3672}
3673
3674static void bpf_iter_udp_put_batch(struct bpf_udp_iter_state *iter)
3675{
3676	union bpf_udp_iter_batch_item *item;
3677	unsigned int cur_sk = iter->cur_sk;
3678	__u64 cookie;
3679
3680	/* Remember the cookies of the sockets we haven't seen yet, so we can
3681	 * pick up where we left off next time around.
3682	 */
3683	while (cur_sk < iter->end_sk) {
3684		item = &iter->batch[cur_sk++];
3685		cookie = sock_gen_cookie(item->sk);
3686		sock_put(item->sk);
3687		item->cookie = cookie;
3688	}
3689}
3690
3691static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v)
3692{
3693	struct bpf_udp_iter_state *iter = seq->private;
3694	struct bpf_iter_meta meta;
3695	struct bpf_prog *prog;
3696
3697	if (!v) {
3698		meta.seq = seq;
3699		prog = bpf_iter_get_info(&meta, true);
3700		if (prog)
3701			(void)udp_prog_seq_show(prog, &meta, v, 0, 0);
3702	}
3703
3704	if (iter->cur_sk < iter->end_sk)
3705		bpf_iter_udp_put_batch(iter);
3706}
3707
3708static const struct seq_operations bpf_iter_udp_seq_ops = {
3709	.start		= bpf_iter_udp_seq_start,
3710	.next		= bpf_iter_udp_seq_next,
3711	.stop		= bpf_iter_udp_seq_stop,
3712	.show		= bpf_iter_udp_seq_show,
3713};
3714#endif
3715
3716static unsigned short seq_file_family(const struct seq_file *seq)
3717{
3718	const struct udp_seq_afinfo *afinfo;
3719
3720#ifdef CONFIG_BPF_SYSCALL
3721	/* BPF iterator: bpf programs to filter sockets. */
3722	if (seq->op == &bpf_iter_udp_seq_ops)
3723		return AF_UNSPEC;
3724#endif
3725
3726	/* Proc fs iterator */
3727	afinfo = pde_data(file_inode(seq->file));
3728	return afinfo->family;
3729}
3730
3731const struct seq_operations udp_seq_ops = {
3732	.start		= udp_seq_start,
3733	.next		= udp_seq_next,
3734	.stop		= udp_seq_stop,
3735	.show		= udp4_seq_show,
3736};
3737EXPORT_IPV6_MOD(udp_seq_ops);
3738
3739static struct udp_seq_afinfo udp4_seq_afinfo = {
3740	.family		= AF_INET,
3741	.udp_table	= NULL,
3742};
3743
3744static int __net_init udp4_proc_init_net(struct net *net)
3745{
3746	if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
3747			sizeof(struct udp_iter_state), &udp4_seq_afinfo))
3748		return -ENOMEM;
3749	return 0;
3750}
3751
3752static void __net_exit udp4_proc_exit_net(struct net *net)
3753{
3754	remove_proc_entry("udp", net->proc_net);
3755}
3756
3757static struct pernet_operations udp4_net_ops = {
3758	.init = udp4_proc_init_net,
3759	.exit = udp4_proc_exit_net,
3760};
3761
3762int __init udp4_proc_init(void)
3763{
3764	return register_pernet_subsys(&udp4_net_ops);
3765}
3766
3767void udp4_proc_exit(void)
3768{
3769	unregister_pernet_subsys(&udp4_net_ops);
3770}
3771#endif /* CONFIG_PROC_FS */
3772
3773static __initdata unsigned long uhash_entries;
3774static int __init set_uhash_entries(char *str)
3775{
3776	ssize_t ret;
3777
3778	if (!str)
3779		return 0;
3780
3781	ret = kstrtoul(str, 0, &uhash_entries);
3782	if (ret)
3783		return 0;
3784
3785	if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
3786		uhash_entries = UDP_HTABLE_SIZE_MIN;
3787	return 1;
3788}
3789__setup("uhash_entries=", set_uhash_entries);
3790
3791void __init udp_table_init(struct udp_table *table, const char *name)
3792{
3793	unsigned int i, slot_size;
3794
3795	slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
3796		    udp_hash4_slot_size();
3797	table->hash = alloc_large_system_hash(name,
3798					      slot_size,
3799					      uhash_entries,
3800					      21, /* one slot per 2 MB */
3801					      0,
3802					      &table->log,
3803					      &table->mask,
3804					      UDP_HTABLE_SIZE_MIN,
3805					      UDP_HTABLE_SIZE_MAX);
3806
3807	table->hash2 = (void *)(table->hash + (table->mask + 1));
3808	for (i = 0; i <= table->mask; i++) {
3809		INIT_HLIST_HEAD(&table->hash[i].head);
3810		table->hash[i].count = 0;
3811		spin_lock_init(&table->hash[i].lock);
3812	}
3813	for (i = 0; i <= table->mask; i++) {
3814		INIT_HLIST_HEAD(&table->hash2[i].hslot.head);
3815		table->hash2[i].hslot.count = 0;
3816		spin_lock_init(&table->hash2[i].hslot.lock);
3817	}
3818	udp_table_hash4_init(table);
3819}
3820
3821u32 udp_flow_hashrnd(void)
3822{
3823	static u32 hashrnd __read_mostly;
3824
3825	net_get_random_once(&hashrnd, sizeof(hashrnd));
3826
3827	return hashrnd;
3828}
3829EXPORT_SYMBOL(udp_flow_hashrnd);
3830
3831static void __net_init udp_sysctl_init(struct net *net)
3832{
3833	net->ipv4.sysctl_udp_rmem_min = PAGE_SIZE;
3834	net->ipv4.sysctl_udp_wmem_min = PAGE_SIZE;
3835
3836#ifdef CONFIG_NET_L3_MASTER_DEV
3837	net->ipv4.sysctl_udp_l3mdev_accept = 0;
3838#endif
3839}
3840
3841static struct udp_table __net_init *udp_pernet_table_alloc(unsigned int hash_entries)
3842{
3843	struct udp_table *udptable;
3844	unsigned int slot_size;
3845	int i;
3846
3847	udptable = kmalloc(sizeof(*udptable), GFP_KERNEL);
3848	if (!udptable)
3849		goto out;
3850
3851	slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main) +
3852		    udp_hash4_slot_size();
3853	udptable->hash = vmalloc_huge(hash_entries * slot_size,
3854				      GFP_KERNEL_ACCOUNT);
3855	if (!udptable->hash)
3856		goto free_table;
3857
3858	udptable->hash2 = (void *)(udptable->hash + hash_entries);
3859	udptable->mask = hash_entries - 1;
3860	udptable->log = ilog2(hash_entries);
3861
3862	for (i = 0; i < hash_entries; i++) {
3863		INIT_HLIST_HEAD(&udptable->hash[i].head);
3864		udptable->hash[i].count = 0;
3865		spin_lock_init(&udptable->hash[i].lock);
3866
3867		INIT_HLIST_HEAD(&udptable->hash2[i].hslot.head);
3868		udptable->hash2[i].hslot.count = 0;
3869		spin_lock_init(&udptable->hash2[i].hslot.lock);
3870	}
3871	udp_table_hash4_init(udptable);
3872
3873	return udptable;
3874
3875free_table:
3876	kfree(udptable);
3877out:
3878	return NULL;
3879}
3880
3881static void __net_exit udp_pernet_table_free(struct net *net)
3882{
3883	struct udp_table *udptable = net->ipv4.udp_table;
3884
3885	if (udptable == &udp_table)
3886		return;
3887
3888	kvfree(udptable->hash);
3889	kfree(udptable);
3890}
3891
3892static void __net_init udp_set_table(struct net *net)
3893{
3894	struct udp_table *udptable;
3895	unsigned int hash_entries;
3896	struct net *old_net;
3897
3898	if (net_eq(net, &init_net))
3899		goto fallback;
3900
3901	old_net = current->nsproxy->net_ns;
3902	hash_entries = READ_ONCE(old_net->ipv4.sysctl_udp_child_hash_entries);
3903	if (!hash_entries)
3904		goto fallback;
3905
3906	/* Set min to keep the bitmap on stack in udp_lib_get_port() */
3907	if (hash_entries < UDP_HTABLE_SIZE_MIN_PERNET)
3908		hash_entries = UDP_HTABLE_SIZE_MIN_PERNET;
3909	else
3910		hash_entries = roundup_pow_of_two(hash_entries);
3911
3912	udptable = udp_pernet_table_alloc(hash_entries);
3913	if (udptable) {
3914		net->ipv4.udp_table = udptable;
3915	} else {
3916		pr_warn("Failed to allocate UDP hash table (entries: %u) "
3917			"for a netns, fallback to the global one\n",
3918			hash_entries);
3919fallback:
3920		net->ipv4.udp_table = &udp_table;
3921	}
3922}
3923
3924static int __net_init udp_pernet_init(struct net *net)
3925{
3926#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
3927	int i;
3928
3929	/* No tunnel is configured */
3930	for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {
3931		INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);
3932		RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);
3933	}
3934#endif
3935	udp_sysctl_init(net);
3936	udp_set_table(net);
3937
3938	return 0;
3939}
3940
3941static void __net_exit udp_pernet_exit(struct net *net)
3942{
3943	udp_pernet_table_free(net);
3944}
3945
3946static struct pernet_operations __net_initdata udp_sysctl_ops = {
3947	.init	= udp_pernet_init,
3948	.exit	= udp_pernet_exit,
3949};
3950
3951#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3952DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
3953		     struct udp_sock *udp_sk, uid_t uid, int bucket)
3954
3955static int bpf_iter_udp_realloc_batch(struct bpf_udp_iter_state *iter,
3956				      unsigned int new_batch_sz, gfp_t flags)
3957{
3958	union bpf_udp_iter_batch_item *new_batch;
3959
3960	new_batch = kvmalloc_array(new_batch_sz, sizeof(*new_batch),
3961				   flags | __GFP_NOWARN);
3962	if (!new_batch)
3963		return -ENOMEM;
3964
3965	if (flags != GFP_NOWAIT)
3966		bpf_iter_udp_put_batch(iter);
3967
3968	memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
3969	kvfree(iter->batch);
3970	iter->batch = new_batch;
3971	iter->max_sk = new_batch_sz;
3972
3973	return 0;
3974}
3975
3976#define INIT_BATCH_SZ 16
3977
3978static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux)
3979{
3980	struct bpf_udp_iter_state *iter = priv_data;
3981	int ret;
3982
3983	ret = bpf_iter_init_seq_net(priv_data, aux);
3984	if (ret)
3985		return ret;
3986
3987	ret = bpf_iter_udp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
3988	if (ret)
3989		bpf_iter_fini_seq_net(priv_data);
3990
3991	iter->state.bucket = -1;
3992
3993	return ret;
3994}
3995
3996static void bpf_iter_fini_udp(void *priv_data)
3997{
3998	struct bpf_udp_iter_state *iter = priv_data;
3999
4000	bpf_iter_fini_seq_net(priv_data);
4001	kvfree(iter->batch);
4002}
4003
4004static const struct bpf_iter_seq_info udp_seq_info = {
4005	.seq_ops		= &bpf_iter_udp_seq_ops,
4006	.init_seq_private	= bpf_iter_init_udp,
4007	.fini_seq_private	= bpf_iter_fini_udp,
4008	.seq_priv_size		= sizeof(struct bpf_udp_iter_state),
4009};
4010
4011static struct bpf_iter_reg udp_reg_info = {
4012	.target			= "udp",
4013	.ctx_arg_info_size	= 1,
4014	.ctx_arg_info		= {
4015		{ offsetof(struct bpf_iter__udp, udp_sk),
4016		  PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
4017	},
4018	.seq_info		= &udp_seq_info,
4019};
4020
4021static void __init bpf_iter_register(void)
4022{
4023	udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
4024	if (bpf_iter_reg_target(&udp_reg_info))
4025		pr_warn("Warning: could not register bpf iterator udp\n");
4026}
4027#endif
4028
4029void __init udp_init(void)
4030{
4031	unsigned long limit;
4032
4033	udp_table_init(&udp_table, "UDP");
4034	limit = nr_free_buffer_pages() / 8;
4035	limit = max(limit, 128UL);
4036	sysctl_udp_mem[0] = limit / 4 * 3;
4037	sysctl_udp_mem[1] = limit;
4038	sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
4039
4040	if (register_pernet_subsys(&udp_sysctl_ops))
4041		panic("UDP: failed to init sysctl parameters.\n");
4042
4043#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
4044	bpf_iter_register();
4045#endif
4046}