tc_tun.c

  1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
  2/* Copyright (c) 2018 Mellanox Technologies. */
  3
  4#include <net/flow.h>
  5#include <net/inet_dscp.h>
  6#include <net/vxlan.h>
  7#include <net/gre.h>
  8#include <net/geneve.h>
  9#include <net/bareudp.h>
 10#include "en/tc_tun.h"
 11#include "en/tc_priv.h"
 12#include "en_tc.h"
 13#include "rep/tc.h"
 14#include "rep/neigh.h"
 15#include "lag/lag.h"
 16#include "lag/mp.h"
 17
 18struct mlx5e_tc_tun_route_attr {
 19	struct net_device *out_dev;
 20	struct net_device *route_dev;
 21	union {
 22		struct flowi4 fl4;
 23		struct flowi6 fl6;
 24	} fl;
 25	struct neighbour *n;
 26	u8 ttl;
 27};
 28
 29#define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {}
 30
 31static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr)
 32{
 33	if (attr->n)
 34		neigh_release(attr->n);
 35	dev_put(attr->route_dev);
 36}
 37
 38struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev)
 39{
 40	if (netif_is_vxlan(tunnel_dev))
 41		return &vxlan_tunnel;
 42	else if (netif_is_geneve(tunnel_dev))
 43		return &geneve_tunnel;
 44	else if (netif_is_gretap(tunnel_dev) ||
 45		 netif_is_ip6gretap(tunnel_dev))
 46		return &gre_tunnel;
 47	else if (netif_is_bareudp(tunnel_dev))
 48		return &mplsoudp_tunnel;
 49	else
 50		return NULL;
 51}
 52
 53static int get_route_and_out_devs(struct mlx5e_priv *priv,
 54				  struct net_device *dev,
 55				  struct net_device **route_dev,
 56				  struct net_device **out_dev)
 57{
 58	struct net_device *uplink_dev, *uplink_upper, *real_dev;
 59	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 60	bool dst_is_lag_dev;
 61
 62	real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev;
 63	uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
 64
 65	rcu_read_lock();
 66	uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev);
 67	/* mlx5_lag_is_sriov() is a blocking function which can't be called
 68	 * while holding rcu read lock. Take the net_device for correctness
 69	 * sake.
 70	 */
 71	dev_hold(uplink_upper);
 72	rcu_read_unlock();
 73
 74	dst_is_lag_dev = (uplink_upper &&
 75			  netif_is_lag_master(uplink_upper) &&
 76			  real_dev == uplink_upper &&
 77			  mlx5_lag_is_sriov(priv->mdev));
 78	dev_put(uplink_upper);
 79
 80	/* if the egress device isn't on the same HW e-switch or
 81	 * it's a LAG device, use the uplink
 82	 */
 83	*route_dev = dev;
 84	if (!netdev_port_same_parent_id(priv->netdev, real_dev) ||
 85	    dst_is_lag_dev || is_vlan_dev(*route_dev) ||
 86	    netif_is_ovs_master(*route_dev))
 87		*out_dev = uplink_dev;
 88	else if (mlx5e_eswitch_rep(dev) &&
 89		 mlx5e_is_valid_eswitch_fwd_dev(priv, dev))
 90		*out_dev = *route_dev;
 91	else
 92		return -EOPNOTSUPP;
 93
 94	if (!mlx5e_eswitch_uplink_rep(*out_dev))
 95		return -EOPNOTSUPP;
 96
 97	if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev &&
 98	    !mlx5_lag_is_mpesw(priv->mdev))
 99		return -EOPNOTSUPP;
100
101	return 0;
102}
103
104static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv,
105				       struct net_device *dev,
106				       struct mlx5e_tc_tun_route_attr *attr)
107{
108	struct net_device *route_dev;
109	struct net_device *out_dev;
110	struct neighbour *n;
111	struct rtable *rt;
112
113#if IS_ENABLED(CONFIG_INET)
114	struct mlx5_core_dev *mdev = priv->mdev;
115	struct net_device *uplink_dev;
116	int ret;
117
118	if (mlx5_lag_is_multipath(mdev)) {
119		struct mlx5_eswitch *esw = mdev->priv.eswitch;
120
121		uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
122		attr->fl.fl4.flowi4_oif = uplink_dev->ifindex;
123	} else {
124		struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
125
126		if (tunnel && tunnel->get_remote_ifindex)
127			attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(dev);
128	}
129
130	rt = ip_route_output_key(dev_net(dev), &attr->fl.fl4);
131	if (IS_ERR(rt))
132		return PTR_ERR(rt);
133
134	if (rt->rt_type != RTN_UNICAST) {
135		ret = -ENETUNREACH;
136		goto err_rt_release;
137	}
138
139	if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) {
140		ret = -ENETUNREACH;
141		goto err_rt_release;
142	}
143#else
144	return -EOPNOTSUPP;
145#endif
146
147	ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev);
148	if (ret < 0)
149		goto err_rt_release;
150	dev_hold(route_dev);
151
152	if (!attr->ttl)
153		attr->ttl = ip4_dst_hoplimit(&rt->dst);
154	n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr);
155	if (!n) {
156		ret = -ENOMEM;
157		goto err_dev_release;
158	}
159
160	ip_rt_put(rt);
161	attr->route_dev = route_dev;
162	attr->out_dev = out_dev;
163	attr->n = n;
164	return 0;
165
166err_dev_release:
167	dev_put(route_dev);
168err_rt_release:
169	ip_rt_put(rt);
170	return ret;
171}
172
173static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr)
174{
175	mlx5e_tc_tun_route_attr_cleanup(attr);
176}
177
178static const char *mlx5e_netdev_kind(struct net_device *dev)
179{
180	if (dev->rtnl_link_ops)
181		return dev->rtnl_link_ops->kind;
182	else
183		return "unknown";
184}
185
186static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
187				      struct mlx5e_encap_entry *e)
188{
189	if (!e->tunnel) {
190		pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
191		return -EOPNOTSUPP;
192	}
193
194	return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
195}
196
197static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
198			     struct mlx5e_encap_entry *e,
199			     u16 proto)
200{
201	struct ethhdr *eth = (struct ethhdr *)buf;
202	char *ip;
203
204	ether_addr_copy(eth->h_dest, e->h_dest);
205	ether_addr_copy(eth->h_source, dev->dev_addr);
206	if (is_vlan_dev(dev)) {
207		struct vlan_hdr *vlan = (struct vlan_hdr *)
208					((char *)eth + ETH_HLEN);
209		ip = (char *)vlan + VLAN_HLEN;
210		eth->h_proto = vlan_dev_vlan_proto(dev);
211		vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
212		vlan->h_vlan_encapsulated_proto = htons(proto);
213	} else {
214		eth->h_proto = htons(proto);
215		ip = (char *)eth + ETH_HLEN;
216	}
217
218	return ip;
219}
220
221int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
222				    struct net_device *mirred_dev,
223				    struct mlx5e_encap_entry *e)
224{
225	int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
226	const struct ip_tunnel_key *tun_key = &e->tun_info->key;
227	struct mlx5_pkt_reformat_params reformat_params;
228	struct mlx5e_neigh m_neigh = {};
229	TC_TUN_ROUTE_ATTR_INIT(attr);
230	int ipv4_encap_size;
231	char *encap_header;
232	struct iphdr *ip;
233	u8 nud_state;
234	int err;
235
236	/* add the IP fields */
237	attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos);
238	attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
239	attr.fl.fl4.saddr = tun_key->u.ipv4.src;
240	attr.ttl = tun_key->ttl;
241
242	err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
243	if (err)
244		return err;
245
246	ipv4_encap_size =
247		(is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
248		sizeof(struct iphdr) +
249		e->tunnel->calc_hlen(e);
250
251	if (max_encap_size < ipv4_encap_size) {
252		mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
253			       ipv4_encap_size, max_encap_size);
254		err = -EOPNOTSUPP;
255		goto release_neigh;
256	}
257
258	encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
259	if (!encap_header) {
260		err = -ENOMEM;
261		goto release_neigh;
262	}
263
264	m_neigh.family = attr.n->ops->family;
265	memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
266	e->out_dev = attr.out_dev;
267	e->route_dev_ifindex = attr.route_dev->ifindex;
268
269	/* It's important to add the neigh to the hash table before checking
270	 * the neigh validity state. So if we'll get a notification, in case the
271	 * neigh changes it's validity state, we would find the relevant neigh
272	 * in the hash.
273	 */
274	err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
275	if (err)
276		goto free_encap;
277
278	read_lock_bh(&attr.n->lock);
279	nud_state = attr.n->nud_state;
280	ether_addr_copy(e->h_dest, attr.n->ha);
281	read_unlock_bh(&attr.n->lock);
282
283	/* add ethernet header */
284	ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
285					     ETH_P_IP);
286
287	/* add ip header */
288	ip->tos = tun_key->tos;
289	ip->version = 0x4;
290	ip->ihl = 0x5;
291	ip->ttl = attr.ttl;
292	ip->daddr = attr.fl.fl4.daddr;
293	ip->saddr = attr.fl.fl4.saddr;
294
295	/* add tunneling protocol header */
296	err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
297					 &ip->protocol, e);
298	if (err)
299		goto destroy_neigh_entry;
300
301	e->encap_size = ipv4_encap_size;
302	e->encap_header = encap_header;
303	encap_header = NULL;
304
305	if (!(nud_state & NUD_VALID)) {
306		neigh_event_send(attr.n, NULL);
307		/* the encap entry will be made valid on neigh update event
308		 * and not used before that.
309		 */
310		goto release_neigh;
311	}
312
313	memset(&reformat_params, 0, sizeof(reformat_params));
314	reformat_params.type = e->reformat_type;
315	reformat_params.size = e->encap_size;
316	reformat_params.data = e->encap_header;
317	e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
318						     MLX5_FLOW_NAMESPACE_FDB);
319	if (IS_ERR(e->pkt_reformat)) {
320		err = PTR_ERR(e->pkt_reformat);
321		goto destroy_neigh_entry;
322	}
323
324	e->flags |= MLX5_ENCAP_ENTRY_VALID;
325	mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
326	mlx5e_route_lookup_ipv4_put(&attr);
327	return err;
328
329destroy_neigh_entry:
330	mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
331free_encap:
332	kfree(encap_header);
333release_neigh:
334	mlx5e_route_lookup_ipv4_put(&attr);
335	return err;
336}
337
338int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
339				    struct net_device *mirred_dev,
340				    struct mlx5e_encap_entry *e)
341{
342	int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
343	const struct ip_tunnel_key *tun_key = &e->tun_info->key;
344	struct mlx5_pkt_reformat_params reformat_params;
345	TC_TUN_ROUTE_ATTR_INIT(attr);
346	int ipv4_encap_size;
347	char *encap_header;
348	struct iphdr *ip;
349	u8 nud_state;
350	int err;
351
352	/* add the IP fields */
353	attr.fl.fl4.flowi4_dscp = inet_dsfield_to_dscp(tun_key->tos);
354	attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
355	attr.fl.fl4.saddr = tun_key->u.ipv4.src;
356	attr.ttl = tun_key->ttl;
357
358	err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
359	if (err)
360		return err;
361
362	ipv4_encap_size =
363		(is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
364		sizeof(struct iphdr) +
365		e->tunnel->calc_hlen(e);
366
367	if (max_encap_size < ipv4_encap_size) {
368		mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
369			       ipv4_encap_size, max_encap_size);
370		err = -EOPNOTSUPP;
371		goto release_neigh;
372	}
373
374	encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
375	if (!encap_header) {
376		err = -ENOMEM;
377		goto release_neigh;
378	}
379
380	e->route_dev_ifindex = attr.route_dev->ifindex;
381
382	read_lock_bh(&attr.n->lock);
383	nud_state = attr.n->nud_state;
384	ether_addr_copy(e->h_dest, attr.n->ha);
385	WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
386	read_unlock_bh(&attr.n->lock);
387
388	/* add ethernet header */
389	ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
390					     ETH_P_IP);
391
392	/* add ip header */
393	ip->tos = tun_key->tos;
394	ip->version = 0x4;
395	ip->ihl = 0x5;
396	ip->ttl = attr.ttl;
397	ip->daddr = attr.fl.fl4.daddr;
398	ip->saddr = attr.fl.fl4.saddr;
399
400	/* add tunneling protocol header */
401	err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
402					 &ip->protocol, e);
403	if (err)
404		goto free_encap;
405
406	e->encap_size = ipv4_encap_size;
407	kfree(e->encap_header);
408	e->encap_header = encap_header;
409	encap_header = NULL;
410
411	if (!(nud_state & NUD_VALID)) {
412		neigh_event_send(attr.n, NULL);
413		/* the encap entry will be made valid on neigh update event
414		 * and not used before that.
415		 */
416		goto release_neigh;
417	}
418
419	memset(&reformat_params, 0, sizeof(reformat_params));
420	reformat_params.type = e->reformat_type;
421	reformat_params.size = e->encap_size;
422	reformat_params.data = e->encap_header;
423	e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
424						     MLX5_FLOW_NAMESPACE_FDB);
425	if (IS_ERR(e->pkt_reformat)) {
426		err = PTR_ERR(e->pkt_reformat);
427		goto free_encap;
428	}
429
430	e->flags |= MLX5_ENCAP_ENTRY_VALID;
431	mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
432	mlx5e_route_lookup_ipv4_put(&attr);
433	return err;
434
435free_encap:
436	kfree(encap_header);
437release_neigh:
438	mlx5e_route_lookup_ipv4_put(&attr);
439	return err;
440}
441
442#if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
443static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv,
444				       struct net_device *dev,
445				       struct mlx5e_tc_tun_route_attr *attr)
446{
447	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev);
448	struct net_device *route_dev;
449	struct net_device *out_dev;
450	struct dst_entry *dst;
451	struct neighbour *n;
452	int ret;
453
454	if (tunnel && tunnel->get_remote_ifindex)
455		attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(dev);
456	dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6,
457					      NULL);
458	if (IS_ERR(dst))
459		return PTR_ERR(dst);
460
461	if (!attr->ttl)
462		attr->ttl = ip6_dst_hoplimit(dst);
463
464	ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev);
465	if (ret < 0)
466		goto err_dst_release;
467
468	dev_hold(route_dev);
469	n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr);
470	if (!n) {
471		ret = -ENOMEM;
472		goto err_dev_release;
473	}
474
475	dst_release(dst);
476	attr->out_dev = out_dev;
477	attr->route_dev = route_dev;
478	attr->n = n;
479	return 0;
480
481err_dev_release:
482	dev_put(route_dev);
483err_dst_release:
484	dst_release(dst);
485	return ret;
486}
487
488static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr)
489{
490	mlx5e_tc_tun_route_attr_cleanup(attr);
491}
492
493int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
494				    struct net_device *mirred_dev,
495				    struct mlx5e_encap_entry *e)
496{
497	int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
498	const struct ip_tunnel_key *tun_key = &e->tun_info->key;
499	struct mlx5_pkt_reformat_params reformat_params;
500	struct mlx5e_neigh m_neigh = {};
501	TC_TUN_ROUTE_ATTR_INIT(attr);
502	struct ipv6hdr *ip6h;
503	int ipv6_encap_size;
504	char *encap_header;
505	u8 nud_state;
506	int err;
507
508	attr.ttl = tun_key->ttl;
509	attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
510	attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
511	attr.fl.fl6.saddr = tun_key->u.ipv6.src;
512
513	err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
514	if (err)
515		return err;
516
517	ipv6_encap_size =
518		(is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
519		sizeof(struct ipv6hdr) +
520		e->tunnel->calc_hlen(e);
521
522	if (max_encap_size < ipv6_encap_size) {
523		mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
524			       ipv6_encap_size, max_encap_size);
525		err = -EOPNOTSUPP;
526		goto release_neigh;
527	}
528
529	encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
530	if (!encap_header) {
531		err = -ENOMEM;
532		goto release_neigh;
533	}
534
535	m_neigh.family = attr.n->ops->family;
536	memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
537	e->out_dev = attr.out_dev;
538	e->route_dev_ifindex = attr.route_dev->ifindex;
539
540	/* It's important to add the neigh to the hash table before checking
541	 * the neigh validity state. So if we'll get a notification, in case the
542	 * neigh changes it's validity state, we would find the relevant neigh
543	 * in the hash.
544	 */
545	err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
546	if (err)
547		goto free_encap;
548
549	read_lock_bh(&attr.n->lock);
550	nud_state = attr.n->nud_state;
551	ether_addr_copy(e->h_dest, attr.n->ha);
552	read_unlock_bh(&attr.n->lock);
553
554	/* add ethernet header */
555	ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
556						 ETH_P_IPV6);
557
558	/* add ip header */
559	ip6_flow_hdr(ip6h, tun_key->tos, 0);
560	/* the HW fills up ipv6 payload len */
561	ip6h->hop_limit   = attr.ttl;
562	ip6h->daddr	  = attr.fl.fl6.daddr;
563	ip6h->saddr	  = attr.fl.fl6.saddr;
564
565	/* add tunneling protocol header */
566	err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
567					 &ip6h->nexthdr, e);
568	if (err)
569		goto destroy_neigh_entry;
570
571	e->encap_size = ipv6_encap_size;
572	e->encap_header = encap_header;
573	encap_header = NULL;
574
575	if (!(nud_state & NUD_VALID)) {
576		neigh_event_send(attr.n, NULL);
577		/* the encap entry will be made valid on neigh update event
578		 * and not used before that.
579		 */
580		goto release_neigh;
581	}
582
583	memset(&reformat_params, 0, sizeof(reformat_params));
584	reformat_params.type = e->reformat_type;
585	reformat_params.size = e->encap_size;
586	reformat_params.data = e->encap_header;
587	e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
588						     MLX5_FLOW_NAMESPACE_FDB);
589	if (IS_ERR(e->pkt_reformat)) {
590		err = PTR_ERR(e->pkt_reformat);
591		goto destroy_neigh_entry;
592	}
593
594	e->flags |= MLX5_ENCAP_ENTRY_VALID;
595	mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
596	mlx5e_route_lookup_ipv6_put(&attr);
597	return err;
598
599destroy_neigh_entry:
600	mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
601free_encap:
602	kfree(encap_header);
603release_neigh:
604	mlx5e_route_lookup_ipv6_put(&attr);
605	return err;
606}
607
608int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv,
609				    struct net_device *mirred_dev,
610				    struct mlx5e_encap_entry *e)
611{
612	int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
613	const struct ip_tunnel_key *tun_key = &e->tun_info->key;
614	struct mlx5_pkt_reformat_params reformat_params;
615	TC_TUN_ROUTE_ATTR_INIT(attr);
616	struct ipv6hdr *ip6h;
617	int ipv6_encap_size;
618	char *encap_header;
619	u8 nud_state;
620	int err;
621
622	attr.ttl = tun_key->ttl;
623
624	attr.fl.fl6.flowlabel = ip6_make_flowinfo(tun_key->tos, tun_key->label);
625	attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
626	attr.fl.fl6.saddr = tun_key->u.ipv6.src;
627
628	err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
629	if (err)
630		return err;
631
632	ipv6_encap_size =
633		(is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
634		sizeof(struct ipv6hdr) +
635		e->tunnel->calc_hlen(e);
636
637	if (max_encap_size < ipv6_encap_size) {
638		mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
639			       ipv6_encap_size, max_encap_size);
640		err = -EOPNOTSUPP;
641		goto release_neigh;
642	}
643
644	encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
645	if (!encap_header) {
646		err = -ENOMEM;
647		goto release_neigh;
648	}
649
650	e->route_dev_ifindex = attr.route_dev->ifindex;
651
652	read_lock_bh(&attr.n->lock);
653	nud_state = attr.n->nud_state;
654	ether_addr_copy(e->h_dest, attr.n->ha);
655	WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
656	read_unlock_bh(&attr.n->lock);
657
658	/* add ethernet header */
659	ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
660						 ETH_P_IPV6);
661
662	/* add ip header */
663	ip6_flow_hdr(ip6h, tun_key->tos, 0);
664	/* the HW fills up ipv6 payload len */
665	ip6h->hop_limit   = attr.ttl;
666	ip6h->daddr	  = attr.fl.fl6.daddr;
667	ip6h->saddr	  = attr.fl.fl6.saddr;
668
669	/* add tunneling protocol header */
670	err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
671					 &ip6h->nexthdr, e);
672	if (err)
673		goto free_encap;
674
675	e->encap_size = ipv6_encap_size;
676	kfree(e->encap_header);
677	e->encap_header = encap_header;
678	encap_header = NULL;
679
680	if (!(nud_state & NUD_VALID)) {
681		neigh_event_send(attr.n, NULL);
682		/* the encap entry will be made valid on neigh update event
683		 * and not used before that.
684		 */
685		goto release_neigh;
686	}
687
688	memset(&reformat_params, 0, sizeof(reformat_params));
689	reformat_params.type = e->reformat_type;
690	reformat_params.size = e->encap_size;
691	reformat_params.data = e->encap_header;
692	e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
693						     MLX5_FLOW_NAMESPACE_FDB);
694	if (IS_ERR(e->pkt_reformat)) {
695		err = PTR_ERR(e->pkt_reformat);
696		goto free_encap;
697	}
698
699	e->flags |= MLX5_ENCAP_ENTRY_VALID;
700	mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
701	mlx5e_route_lookup_ipv6_put(&attr);
702	return err;
703
704free_encap:
705	kfree(encap_header);
706release_neigh:
707	mlx5e_route_lookup_ipv6_put(&attr);
708	return err;
709}
710#endif
711
712int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv,
713			      struct mlx5_flow_spec *spec,
714			      struct mlx5_flow_attr *flow_attr,
715			      struct net_device *filter_dev)
716{
717	struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr;
718	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
719	struct mlx5e_tc_int_port *int_port;
720	TC_TUN_ROUTE_ATTR_INIT(attr);
721	u16 vport_num;
722	int err = 0;
723
724	if (flow_attr->tun_ip_version == 4) {
725		/* Addresses are swapped for decap */
726		attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4;
727		attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4;
728		err = mlx5e_route_lookup_ipv4_get(priv, filter_dev, &attr);
729	}
730#if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
731	else if (flow_attr->tun_ip_version == 6) {
732		/* Addresses are swapped for decap */
733		attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6;
734		attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6;
735		err = mlx5e_route_lookup_ipv6_get(priv, filter_dev, &attr);
736	}
737#endif
738	else
739		return 0;
740
741	if (err)
742		return err;
743
744	if (attr.route_dev->netdev_ops == &mlx5e_netdev_ops &&
745	    mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev)) {
746		err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num);
747		if (err)
748			goto out;
749
750		esw_attr->rx_tun_attr->decap_vport = vport_num;
751	} else if (netif_is_ovs_master(attr.route_dev) && mlx5e_tc_int_port_supported(esw)) {
752		int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv),
753						 attr.route_dev->ifindex,
754						 MLX5E_TC_INT_PORT_INGRESS);
755		if (IS_ERR(int_port)) {
756			err = PTR_ERR(int_port);
757			goto out;
758		}
759		esw_attr->int_port = int_port;
760	}
761
762out:
763	if (flow_attr->tun_ip_version == 4)
764		mlx5e_route_lookup_ipv4_put(&attr);
765#if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
766	else if (flow_attr->tun_ip_version == 6)
767		mlx5e_route_lookup_ipv6_put(&attr);
768#endif
769	return err;
770}
771
772bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
773				    struct net_device *netdev)
774{
775	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);
776
777	if (tunnel && tunnel->can_offload(priv))
778		return true;
779	else
780		return false;
781}
782
783int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
784				 struct mlx5e_priv *priv,
785				 struct mlx5e_encap_entry *e,
786				 struct netlink_ext_ack *extack)
787{
788	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev);
789
790	if (!tunnel) {
791		e->reformat_type = -1;
792		return -EOPNOTSUPP;
793	}
794
795	return tunnel->init_encap_attr(tunnel_dev, priv, e, extack);
796}
797
798int mlx5e_tc_tun_parse(struct net_device *filter_dev,
799		       struct mlx5e_priv *priv,
800		       struct mlx5_flow_spec *spec,
801		       struct flow_cls_offload *f,
802		       u8 *match_level)
803{
804	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev);
805	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
806	void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
807				       outer_headers);
808	void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
809				       outer_headers);
810	struct netlink_ext_ack *extack = f->common.extack;
811	int err = 0;
812
813	if (!tunnel) {
814		netdev_warn(priv->netdev,
815			    "decapsulation offload is not supported for %s net device\n",
816			    mlx5e_netdev_kind(filter_dev));
817		err = -EOPNOTSUPP;
818		goto out;
819	}
820
821	*match_level = tunnel->match_level;
822
823	if (tunnel->parse_udp_ports) {
824		err = tunnel->parse_udp_ports(priv, spec, f,
825					      headers_c, headers_v);
826		if (err)
827			goto out;
828	}
829
830	if (tunnel->parse_tunnel) {
831		err = tunnel->parse_tunnel(priv, spec, f,
832					   headers_c, headers_v);
833		if (err)
834			goto out;
835	}
836
837	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
838		struct flow_dissector_key_basic key_basic = {};
839		struct flow_dissector_key_basic mask_basic = {
840			.n_proto = htons(0xFFFF),
841		};
842		struct flow_match_basic match_basic = {
843			.key = &key_basic, .mask = &mask_basic,
844		};
845		struct flow_match_control match;
846		u16 addr_type;
847
848		flow_rule_match_enc_control(rule, &match);
849		addr_type = match.key->addr_type;
850
851		if (flow_rule_has_enc_control_flags(match.mask->flags,
852						    extack)) {
853			err = -EOPNOTSUPP;
854			goto out;
855		}
856
857		/* For tunnel addr_type used same key id`s as for non-tunnel */
858		if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
859			struct flow_match_ipv4_addrs match;
860
861			flow_rule_match_enc_ipv4_addrs(rule, &match);
862			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
863				 src_ipv4_src_ipv6.ipv4_layout.ipv4,
864				 ntohl(match.mask->src));
865			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
866				 src_ipv4_src_ipv6.ipv4_layout.ipv4,
867				 ntohl(match.key->src));
868
869			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
870				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
871				 ntohl(match.mask->dst));
872			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
873				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
874				 ntohl(match.key->dst));
875
876			key_basic.n_proto = htons(ETH_P_IP);
877			mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
878					       headers_c, headers_v);
879		} else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
880			struct flow_match_ipv6_addrs match;
881
882			flow_rule_match_enc_ipv6_addrs(rule, &match);
883			memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
884					    src_ipv4_src_ipv6.ipv6_layout.ipv6),
885			       &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
886								   ipv6));
887			memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
888					    src_ipv4_src_ipv6.ipv6_layout.ipv6),
889			       &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
890								  ipv6));
891
892			memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
893					    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
894			       &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
895								   ipv6));
896			memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
897					    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
898			       &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
899								  ipv6));
900
901			key_basic.n_proto = htons(ETH_P_IPV6);
902			mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
903					       headers_c, headers_v);
904		}
905	}
906
907	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
908		struct flow_match_ip match;
909
910		flow_rule_match_enc_ip(rule, &match);
911		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
912			 match.mask->tos & 0x3);
913		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
914			 match.key->tos & 0x3);
915
916		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
917			 match.mask->tos >> 2);
918		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
919			 match.key->tos  >> 2);
920
921		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
922			 match.mask->ttl);
923		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
924			 match.key->ttl);
925
926		if (match.mask->ttl &&
927		    !MLX5_CAP_ESW_FLOWTABLE_FDB
928			(priv->mdev,
929			 ft_field_support.outer_ipv4_ttl)) {
930			NL_SET_ERR_MSG_MOD(extack,
931					   "Matching on TTL is not supported");
932			err = -EOPNOTSUPP;
933			goto out;
934		}
935	}
936
937	/* let software handle IP fragments */
938	MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
939	MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
940
941	return 0;
942
943out:
944	return err;
945}
946
947int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
948				 struct mlx5_flow_spec *spec,
949				 struct flow_cls_offload *f,
950				 void *headers_c,
951				 void *headers_v)
952{
953	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
954	struct netlink_ext_ack *extack = f->common.extack;
955	struct flow_match_ports enc_ports;
956
957	/* Full udp dst port must be given */
958
959	if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
960		NL_SET_ERR_MSG_MOD(extack,
961				   "UDP tunnel decap filter must include enc_dst_port condition");
962		netdev_warn(priv->netdev,
963			    "UDP tunnel decap filter must include enc_dst_port condition\n");
964		return -EOPNOTSUPP;
965	}
966
967	flow_rule_match_enc_ports(rule, &enc_ports);
968
969	if (memchr_inv(&enc_ports.mask->dst, 0xff,
970		       sizeof(enc_ports.mask->dst))) {
971		NL_SET_ERR_MSG_MOD(extack,
972				   "UDP tunnel decap filter must match enc_dst_port fully");
973		netdev_warn(priv->netdev,
974			    "UDP tunnel decap filter must match enc_dst_port fully\n");
975		return -EOPNOTSUPP;
976	}
977
978	/* match on UDP protocol and dst port number */
979
980	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
981	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
982
983	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
984		 ntohs(enc_ports.mask->dst));
985	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
986		 ntohs(enc_ports.key->dst));
987
988	/* UDP src port on outer header is generated by HW,
989	 * so it is probably a bad idea to request matching it.
990	 * Nonetheless, it is allowed.
991	 */
992
993	MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
994		 ntohs(enc_ports.mask->src));
995	MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
996		 ntohs(enc_ports.key->src));
997
998	return 0;
999}