tcpv6_offload.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 *	IPV6 GSO/GRO offload support
  4 *	Linux INET6 implementation
  5 *
  6 *      TCPv6 GSO/GRO support
  7 */
  8#include <linux/indirect_call_wrapper.h>
  9#include <linux/skbuff.h>
 10#include <net/inet6_hashtables.h>
 11#include <net/gro.h>
 12#include <net/protocol.h>
 13#include <net/tcp.h>
 14#include <net/ip6_checksum.h>
 15#include "ip6_offload.h"
 16
 17static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
 18				    struct tcphdr *th)
 19{
 20#if IS_ENABLED(CONFIG_IPV6)
 21	const struct ipv6hdr *hdr;
 22	struct sk_buff *p;
 23	struct sock *sk;
 24	struct net *net;
 25	int iif, sdif;
 26
 27	if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST)))
 28		return;
 29
 30	p = tcp_gro_lookup(head, th);
 31	if (p) {
 32		NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
 33		return;
 34	}
 35
 36	inet6_get_iif_sdif(skb, &iif, &sdif);
 37	hdr = skb_gro_network_header(skb);
 38	net = dev_net_rcu(skb->dev);
 39	sk = __inet6_lookup_established(net, &hdr->saddr, th->source,
 40					&hdr->daddr, ntohs(th->dest),
 41					iif, sdif);
 42	NAPI_GRO_CB(skb)->is_flist = !sk;
 43	if (sk)
 44		sock_gen_put(sk);
 45#endif /* IS_ENABLED(CONFIG_IPV6) */
 46}
 47
 48INDIRECT_CALLABLE_SCOPE
 49struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
 50{
 51	struct tcphdr *th;
 52
 53	/* Don't bother verifying checksum if we're going to flush anyway. */
 54	if (!NAPI_GRO_CB(skb)->flush &&
 55	    skb_gro_checksum_validate(skb, IPPROTO_TCP,
 56				      ip6_gro_compute_pseudo))
 57		goto flush;
 58
 59	th = tcp_gro_pull_header(skb);
 60	if (!th)
 61		goto flush;
 62
 63	tcp6_check_fraglist_gro(head, skb, th);
 64
 65	return tcp_gro_receive(head, skb, th);
 66
 67flush:
 68	NAPI_GRO_CB(skb)->flush = 1;
 69	return NULL;
 70}
 71
 72INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
 73{
 74	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
 75	const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
 76	struct tcphdr *th = tcp_hdr(skb);
 77
 78	if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
 79		skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
 80		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
 81
 82		__skb_incr_checksum_unnecessary(skb);
 83
 84		return 0;
 85	}
 86
 87	th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
 88				  &iph->daddr, 0);
 89	skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
 90
 91	tcp_gro_complete(skb);
 92	return 0;
 93}
 94
 95static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
 96				     struct in6_addr *oldip,
 97				     const struct in6_addr *newip,
 98				     __be16 *oldport, __be16 newport)
 99{
100	struct tcphdr *th = tcp_hdr(seg);
101
102	if (!ipv6_addr_equal(oldip, newip)) {
103		inet_proto_csum_replace16(&th->check, seg,
104					  oldip->s6_addr32,
105					  newip->s6_addr32,
106					  true);
107		*oldip = *newip;
108	}
109
110	if (*oldport == newport)
111		return;
112
113	inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
114	*oldport = newport;
115}
116
117static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
118{
119	const struct tcphdr *th;
120	const struct ipv6hdr *iph;
121	struct sk_buff *seg;
122	struct tcphdr *th2;
123	struct ipv6hdr *iph2;
124
125	seg = segs;
126	th = tcp_hdr(seg);
127	iph = ipv6_hdr(seg);
128	th2 = tcp_hdr(seg->next);
129	iph2 = ipv6_hdr(seg->next);
130
131	if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
132	    ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
133	    ipv6_addr_equal(&iph->daddr, &iph2->daddr))
134		return segs;
135
136	while ((seg = seg->next)) {
137		th2 = tcp_hdr(seg);
138		iph2 = ipv6_hdr(seg);
139
140		__tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr,
141					 &th2->source, th->source);
142		__tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr,
143					 &th2->dest, th->dest);
144	}
145
146	return segs;
147}
148
149static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
150					      netdev_features_t features)
151{
152	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
153	if (IS_ERR(skb))
154		return skb;
155
156	return __tcpv6_gso_segment_list_csum(skb);
157}
158
159static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
160					netdev_features_t features)
161{
162	struct tcphdr *th;
163
164	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
165		return ERR_PTR(-EINVAL);
166
167	if (!pskb_may_pull(skb, sizeof(*th)))
168		return ERR_PTR(-EINVAL);
169
170	if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
171		struct tcphdr *th = tcp_hdr(skb);
172
173		if ((skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size) &&
174		    !(skb_shinfo(skb)->gso_type & SKB_GSO_DODGY))
175			return __tcp6_gso_segment_list(skb, features);
176
177		skb->ip_summed = CHECKSUM_NONE;
178	}
179
180	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
181		const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
182		struct tcphdr *th = tcp_hdr(skb);
183
184		/* Set up pseudo header, usually expect stack to have done
185		 * this.
186		 */
187
188		th->check = 0;
189		skb->ip_summed = CHECKSUM_PARTIAL;
190		__tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr);
191	}
192
193	return tcp_gso_segment(skb, features);
194}
195
196int __init tcpv6_offload_init(void)
197{
198	net_hotdata.tcpv6_offload = (struct net_offload) {
199		.callbacks = {
200			.gso_segment	=	tcp6_gso_segment,
201			.gro_receive	=	tcp6_gro_receive,
202			.gro_complete	=	tcp6_gro_complete,
203		},
204	};
205	return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
206}