en_fs_ethtool.c

   1/*
   2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/mlx5/fs.h>
  34#include "en.h"
  35#include "en/params.h"
  36#include "en/xsk/pool.h"
  37#include "en/fs_ethtool.h"
  38
  39struct mlx5e_ethtool_table {
  40	struct mlx5_flow_table *ft;
  41	int                    num_rules;
  42};
  43
  44#define ETHTOOL_NUM_L3_L4_FTS 7
  45#define ETHTOOL_NUM_L2_FTS 4
  46
  47struct mlx5e_ethtool_steering {
  48	struct mlx5e_ethtool_table      l3_l4_ft[ETHTOOL_NUM_L3_L4_FTS];
  49	struct mlx5e_ethtool_table      l2_ft[ETHTOOL_NUM_L2_FTS];
  50	struct list_head                rules;
  51	int                             tot_num_rules;
  52};
  53
  54static int flow_type_to_traffic_type(u32 flow_type);
  55
  56static u32 flow_type_mask(u32 flow_type)
  57{
  58	return flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS);
  59}
  60
  61struct mlx5e_ethtool_rule {
  62	struct list_head             list;
  63	struct ethtool_rx_flow_spec  flow_spec;
  64	struct mlx5_flow_handle	     *rule;
  65	struct mlx5e_ethtool_table   *eth_ft;
  66	struct mlx5e_rss             *rss;
  67};
  68
  69static void put_flow_table(struct mlx5e_ethtool_table *eth_ft)
  70{
  71	if (!--eth_ft->num_rules) {
  72		mlx5_destroy_flow_table(eth_ft->ft);
  73		eth_ft->ft = NULL;
  74	}
  75}
  76
  77#define MLX5E_ETHTOOL_L3_L4_PRIO 0
  78#define MLX5E_ETHTOOL_L2_PRIO (MLX5E_ETHTOOL_L3_L4_PRIO + ETHTOOL_NUM_L3_L4_FTS)
  79#define MLX5E_ETHTOOL_NUM_ENTRIES 64000
  80#define MLX5E_ETHTOOL_NUM_GROUPS  10
  81static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv,
  82						  struct ethtool_rx_flow_spec *fs,
  83						  int num_tuples)
  84{
  85	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
  86	struct mlx5_flow_table_attr ft_attr = {};
  87	struct mlx5e_ethtool_table *eth_ft;
  88	struct mlx5_flow_namespace *ns;
  89	struct mlx5_flow_table *ft;
  90	int max_tuples;
  91	int table_size;
  92	int prio;
  93
  94	switch (flow_type_mask(fs->flow_type)) {
  95	case TCP_V4_FLOW:
  96	case UDP_V4_FLOW:
  97	case TCP_V6_FLOW:
  98	case UDP_V6_FLOW:
  99	case IP_USER_FLOW:
 100	case IPV6_USER_FLOW:
 101		max_tuples = ETHTOOL_NUM_L3_L4_FTS;
 102		prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples);
 103		eth_ft = &ethtool->l3_l4_ft[prio];
 104		break;
 105	case ETHER_FLOW:
 106		max_tuples = ETHTOOL_NUM_L2_FTS;
 107		prio = max_tuples - num_tuples;
 108		eth_ft = &ethtool->l2_ft[prio];
 109		prio += MLX5E_ETHTOOL_L2_PRIO;
 110		break;
 111	default:
 112		return ERR_PTR(-EINVAL);
 113	}
 114
 115	eth_ft->num_rules++;
 116	if (eth_ft->ft)
 117		return eth_ft;
 118
 119	ns = mlx5_get_flow_namespace(priv->mdev,
 120				     MLX5_FLOW_NAMESPACE_ETHTOOL);
 121	if (!ns)
 122		return ERR_PTR(-EOPNOTSUPP);
 123
 124	table_size = min_t(u32, BIT(MLX5_CAP_FLOWTABLE(priv->mdev,
 125						       flow_table_properties_nic_receive.log_max_ft_size)),
 126			   MLX5E_ETHTOOL_NUM_ENTRIES);
 127
 128	ft_attr.prio = prio;
 129	ft_attr.max_fte = table_size;
 130	ft_attr.autogroup.max_num_groups = MLX5E_ETHTOOL_NUM_GROUPS;
 131	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
 132	if (IS_ERR(ft))
 133		return (void *)ft;
 134
 135	eth_ft->ft = ft;
 136	return eth_ft;
 137}
 138
 139static void mask_spec(u8 *mask, u8 *val, size_t size)
 140{
 141	unsigned int i;
 142
 143	for (i = 0; i < size; i++, mask++, val++)
 144		*((u8 *)val) = *((u8 *)mask) & *((u8 *)val);
 145}
 146
 147#define MLX5E_FTE_SET(header_p, fld, v)  \
 148	MLX5_SET(fte_match_set_lyr_2_4, header_p, fld, v)
 149
 150#define MLX5E_FTE_ADDR_OF(header_p, fld) \
 151	MLX5_ADDR_OF(fte_match_set_lyr_2_4, header_p, fld)
 152
 153static void
 154set_ip4(void *headers_c, void *headers_v, __be32 ip4src_m,
 155	__be32 ip4src_v, __be32 ip4dst_m, __be32 ip4dst_v)
 156{
 157	if (ip4src_m) {
 158		memcpy(MLX5E_FTE_ADDR_OF(headers_v, src_ipv4_src_ipv6.ipv4_layout.ipv4),
 159		       &ip4src_v, sizeof(ip4src_v));
 160		memcpy(MLX5E_FTE_ADDR_OF(headers_c, src_ipv4_src_ipv6.ipv4_layout.ipv4),
 161		       &ip4src_m, sizeof(ip4src_m));
 162	}
 163	if (ip4dst_m) {
 164		memcpy(MLX5E_FTE_ADDR_OF(headers_v, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 165		       &ip4dst_v, sizeof(ip4dst_v));
 166		memcpy(MLX5E_FTE_ADDR_OF(headers_c, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 167		       &ip4dst_m, sizeof(ip4dst_m));
 168	}
 169
 170	MLX5E_FTE_SET(headers_c, ethertype, 0xffff);
 171	MLX5E_FTE_SET(headers_v, ethertype, ETH_P_IP);
 172}
 173
 174static void
 175set_ip6(void *headers_c, void *headers_v, __be32 ip6src_m[4],
 176	__be32 ip6src_v[4], __be32 ip6dst_m[4], __be32 ip6dst_v[4])
 177{
 178	u8 ip6_sz = MLX5_FLD_SZ_BYTES(ipv6_layout, ipv6);
 179
 180	if (!ipv6_addr_any((struct in6_addr *)ip6src_m)) {
 181		memcpy(MLX5E_FTE_ADDR_OF(headers_v, src_ipv4_src_ipv6.ipv6_layout.ipv6),
 182		       ip6src_v, ip6_sz);
 183		memcpy(MLX5E_FTE_ADDR_OF(headers_c, src_ipv4_src_ipv6.ipv6_layout.ipv6),
 184		       ip6src_m, ip6_sz);
 185	}
 186	if (!ipv6_addr_any((struct in6_addr *)ip6dst_m)) {
 187		memcpy(MLX5E_FTE_ADDR_OF(headers_v, dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 188		       ip6dst_v, ip6_sz);
 189		memcpy(MLX5E_FTE_ADDR_OF(headers_c, dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 190		       ip6dst_m, ip6_sz);
 191	}
 192
 193	MLX5E_FTE_SET(headers_c, ethertype, 0xffff);
 194	MLX5E_FTE_SET(headers_v, ethertype, ETH_P_IPV6);
 195}
 196
 197static void
 198set_tcp(void *headers_c, void *headers_v, __be16 psrc_m, __be16 psrc_v,
 199	__be16 pdst_m, __be16 pdst_v)
 200{
 201	if (psrc_m) {
 202		MLX5E_FTE_SET(headers_c, tcp_sport, ntohs(psrc_m));
 203		MLX5E_FTE_SET(headers_v, tcp_sport, ntohs(psrc_v));
 204	}
 205	if (pdst_m) {
 206		MLX5E_FTE_SET(headers_c, tcp_dport, ntohs(pdst_m));
 207		MLX5E_FTE_SET(headers_v, tcp_dport, ntohs(pdst_v));
 208	}
 209
 210	MLX5E_FTE_SET(headers_c, ip_protocol, 0xffff);
 211	MLX5E_FTE_SET(headers_v, ip_protocol, IPPROTO_TCP);
 212}
 213
 214static void
 215set_udp(void *headers_c, void *headers_v, __be16 psrc_m, __be16 psrc_v,
 216	__be16 pdst_m, __be16 pdst_v)
 217{
 218	if (psrc_m) {
 219		MLX5E_FTE_SET(headers_c, udp_sport, ntohs(psrc_m));
 220		MLX5E_FTE_SET(headers_v, udp_sport, ntohs(psrc_v));
 221	}
 222
 223	if (pdst_m) {
 224		MLX5E_FTE_SET(headers_c, udp_dport, ntohs(pdst_m));
 225		MLX5E_FTE_SET(headers_v, udp_dport, ntohs(pdst_v));
 226	}
 227
 228	MLX5E_FTE_SET(headers_c, ip_protocol, 0xffff);
 229	MLX5E_FTE_SET(headers_v, ip_protocol, IPPROTO_UDP);
 230}
 231
 232static void
 233parse_tcp4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 234{
 235	struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.tcp_ip4_spec;
 236	struct ethtool_tcpip4_spec *l4_val  = &fs->h_u.tcp_ip4_spec;
 237
 238	set_ip4(headers_c, headers_v, l4_mask->ip4src, l4_val->ip4src,
 239		l4_mask->ip4dst, l4_val->ip4dst);
 240
 241	set_tcp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
 242		l4_mask->pdst, l4_val->pdst);
 243}
 244
 245static void
 246parse_udp4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 247{
 248	struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.udp_ip4_spec;
 249	struct ethtool_tcpip4_spec *l4_val  = &fs->h_u.udp_ip4_spec;
 250
 251	set_ip4(headers_c, headers_v, l4_mask->ip4src, l4_val->ip4src,
 252		l4_mask->ip4dst, l4_val->ip4dst);
 253
 254	set_udp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
 255		l4_mask->pdst, l4_val->pdst);
 256}
 257
 258static void
 259parse_ip4(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 260{
 261	struct ethtool_usrip4_spec *l3_mask = &fs->m_u.usr_ip4_spec;
 262	struct ethtool_usrip4_spec *l3_val  = &fs->h_u.usr_ip4_spec;
 263
 264	set_ip4(headers_c, headers_v, l3_mask->ip4src, l3_val->ip4src,
 265		l3_mask->ip4dst, l3_val->ip4dst);
 266
 267	if (l3_mask->proto) {
 268		MLX5E_FTE_SET(headers_c, ip_protocol, l3_mask->proto);
 269		MLX5E_FTE_SET(headers_v, ip_protocol, l3_val->proto);
 270	}
 271}
 272
 273static void
 274parse_ip6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 275{
 276	struct ethtool_usrip6_spec *l3_mask = &fs->m_u.usr_ip6_spec;
 277	struct ethtool_usrip6_spec *l3_val  = &fs->h_u.usr_ip6_spec;
 278
 279	set_ip6(headers_c, headers_v, l3_mask->ip6src,
 280		l3_val->ip6src, l3_mask->ip6dst, l3_val->ip6dst);
 281
 282	if (l3_mask->l4_proto) {
 283		MLX5E_FTE_SET(headers_c, ip_protocol, l3_mask->l4_proto);
 284		MLX5E_FTE_SET(headers_v, ip_protocol, l3_val->l4_proto);
 285	}
 286}
 287
 288static void
 289parse_tcp6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 290{
 291	struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.tcp_ip6_spec;
 292	struct ethtool_tcpip6_spec *l4_val  = &fs->h_u.tcp_ip6_spec;
 293
 294	set_ip6(headers_c, headers_v, l4_mask->ip6src,
 295		l4_val->ip6src, l4_mask->ip6dst, l4_val->ip6dst);
 296
 297	set_tcp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
 298		l4_mask->pdst, l4_val->pdst);
 299}
 300
 301static void
 302parse_udp6(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 303{
 304	struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.udp_ip6_spec;
 305	struct ethtool_tcpip6_spec *l4_val  = &fs->h_u.udp_ip6_spec;
 306
 307	set_ip6(headers_c, headers_v, l4_mask->ip6src,
 308		l4_val->ip6src, l4_mask->ip6dst, l4_val->ip6dst);
 309
 310	set_udp(headers_c, headers_v, l4_mask->psrc, l4_val->psrc,
 311		l4_mask->pdst, l4_val->pdst);
 312}
 313
 314static void
 315parse_ether(void *headers_c, void *headers_v, struct ethtool_rx_flow_spec *fs)
 316{
 317	struct ethhdr *eth_mask = &fs->m_u.ether_spec;
 318	struct ethhdr *eth_val = &fs->h_u.ether_spec;
 319
 320	mask_spec((u8 *)eth_mask, (u8 *)eth_val, sizeof(*eth_mask));
 321	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, smac_47_16), eth_mask->h_source);
 322	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, smac_47_16), eth_val->h_source);
 323	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, dmac_47_16), eth_mask->h_dest);
 324	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, dmac_47_16), eth_val->h_dest);
 325	MLX5E_FTE_SET(headers_c, ethertype, ntohs(eth_mask->h_proto));
 326	MLX5E_FTE_SET(headers_v, ethertype, ntohs(eth_val->h_proto));
 327}
 328
 329static void
 330set_cvlan(void *headers_c, void *headers_v, __be16 vlan_tci)
 331{
 332	MLX5E_FTE_SET(headers_c, cvlan_tag, 1);
 333	MLX5E_FTE_SET(headers_v, cvlan_tag, 1);
 334	MLX5E_FTE_SET(headers_c, first_vid, 0xfff);
 335	MLX5E_FTE_SET(headers_v, first_vid, ntohs(vlan_tci));
 336}
 337
 338static void
 339set_dmac(void *headers_c, void *headers_v,
 340	 unsigned char m_dest[ETH_ALEN], unsigned char v_dest[ETH_ALEN])
 341{
 342	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_c, dmac_47_16), m_dest);
 343	ether_addr_copy(MLX5E_FTE_ADDR_OF(headers_v, dmac_47_16), v_dest);
 344}
 345
 346static int set_flow_attrs(u32 *match_c, u32 *match_v,
 347			  struct ethtool_rx_flow_spec *fs)
 348{
 349	void *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
 350					     outer_headers);
 351	void *outer_headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
 352					     outer_headers);
 353	u32 flow_type = flow_type_mask(fs->flow_type);
 354
 355	switch (flow_type) {
 356	case TCP_V4_FLOW:
 357		parse_tcp4(outer_headers_c, outer_headers_v, fs);
 358		break;
 359	case UDP_V4_FLOW:
 360		parse_udp4(outer_headers_c, outer_headers_v, fs);
 361		break;
 362	case IP_USER_FLOW:
 363		parse_ip4(outer_headers_c, outer_headers_v, fs);
 364		break;
 365	case TCP_V6_FLOW:
 366		parse_tcp6(outer_headers_c, outer_headers_v, fs);
 367		break;
 368	case UDP_V6_FLOW:
 369		parse_udp6(outer_headers_c, outer_headers_v, fs);
 370		break;
 371	case IPV6_USER_FLOW:
 372		parse_ip6(outer_headers_c, outer_headers_v, fs);
 373		break;
 374	case ETHER_FLOW:
 375		parse_ether(outer_headers_c, outer_headers_v, fs);
 376		break;
 377	default:
 378		return -EINVAL;
 379	}
 380
 381	if ((fs->flow_type & FLOW_EXT) &&
 382	    (fs->m_ext.vlan_tci & cpu_to_be16(VLAN_VID_MASK)))
 383		set_cvlan(outer_headers_c, outer_headers_v, fs->h_ext.vlan_tci);
 384
 385	if (fs->flow_type & FLOW_MAC_EXT &&
 386	    !is_zero_ether_addr(fs->m_ext.h_dest)) {
 387		mask_spec(fs->m_ext.h_dest, fs->h_ext.h_dest, ETH_ALEN);
 388		set_dmac(outer_headers_c, outer_headers_v, fs->m_ext.h_dest,
 389			 fs->h_ext.h_dest);
 390	}
 391
 392	return 0;
 393}
 394
 395static void add_rule_to_list(struct mlx5e_priv *priv,
 396			     struct mlx5e_ethtool_rule *rule)
 397{
 398	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 399	struct list_head *head = &ethtool->rules;
 400	struct mlx5e_ethtool_rule *iter;
 401
 402	list_for_each_entry(iter, &ethtool->rules, list) {
 403		if (iter->flow_spec.location > rule->flow_spec.location)
 404			break;
 405		head = &iter->list;
 406	}
 407	ethtool->tot_num_rules++;
 408	list_add(&rule->list, head);
 409}
 410
 411static bool outer_header_zero(u32 *match_criteria)
 412{
 413	int size = MLX5_FLD_SZ_BYTES(fte_match_param, outer_headers);
 414	char *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_criteria,
 415					     outer_headers);
 416
 417	return outer_headers_c[0] == 0 && !memcmp(outer_headers_c,
 418						  outer_headers_c + 1,
 419						  size - 1);
 420}
 421
 422static int flow_get_tirn(struct mlx5e_priv *priv,
 423			 struct mlx5e_ethtool_rule *eth_rule,
 424			 struct ethtool_rx_flow_spec *fs,
 425			 u32 rss_context, u32 *tirn)
 426{
 427	if (fs->flow_type & FLOW_RSS) {
 428		struct mlx5e_packet_merge_param pkt_merge_param;
 429		struct mlx5e_rss *rss;
 430		u32 flow_type;
 431		int err;
 432		int tt;
 433
 434		rss = mlx5e_rx_res_rss_get(priv->rx_res, rss_context);
 435		if (!rss)
 436			return -ENOENT;
 437
 438		flow_type = flow_type_mask(fs->flow_type);
 439		tt = flow_type_to_traffic_type(flow_type);
 440		if (tt < 0)
 441			return -EINVAL;
 442
 443		pkt_merge_param = priv->channels.params.packet_merge;
 444		err = mlx5e_rss_obtain_tirn(rss, tt, &pkt_merge_param, false, tirn);
 445		if (err)
 446			return err;
 447		eth_rule->rss = rss;
 448		mlx5e_rss_refcnt_inc(eth_rule->rss);
 449	} else {
 450		*tirn = mlx5e_rx_res_get_tirn_direct(priv->rx_res, fs->ring_cookie);
 451	}
 452
 453	return 0;
 454}
 455
 456static struct mlx5_flow_handle *
 457add_ethtool_flow_rule(struct mlx5e_priv *priv,
 458		      struct mlx5e_ethtool_rule *eth_rule,
 459		      struct mlx5_flow_table *ft,
 460		      struct ethtool_rx_flow_spec *fs, u32 rss_context)
 461{
 462	struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND };
 463	struct mlx5_flow_destination *dst = NULL;
 464	struct mlx5_flow_handle *rule;
 465	struct mlx5_flow_spec *spec;
 466	int err = 0;
 467
 468	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 469	if (!spec)
 470		return ERR_PTR(-ENOMEM);
 471	err = set_flow_attrs(spec->match_criteria, spec->match_value,
 472			     fs);
 473	if (err)
 474		goto free;
 475
 476	if (fs->ring_cookie == RX_CLS_FLOW_DISC) {
 477		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
 478	} else {
 479		dst = kzalloc(sizeof(*dst), GFP_KERNEL);
 480		if (!dst) {
 481			err = -ENOMEM;
 482			goto free;
 483		}
 484
 485		err = flow_get_tirn(priv, eth_rule, fs, rss_context, &dst->tir_num);
 486		if (err)
 487			goto free;
 488
 489		dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 490		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 491	}
 492
 493	spec->match_criteria_enable = (!outer_header_zero(spec->match_criteria));
 494	spec->flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
 495	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dst, dst ? 1 : 0);
 496	if (IS_ERR(rule)) {
 497		err = PTR_ERR(rule);
 498		netdev_err(priv->netdev, "%s: failed to add ethtool steering rule: %d\n",
 499			   __func__, err);
 500		goto free;
 501	}
 502free:
 503	kvfree(spec);
 504	kfree(dst);
 505	return err ? ERR_PTR(err) : rule;
 506}
 507
 508static void del_ethtool_rule(struct mlx5e_flow_steering *fs,
 509			     struct mlx5e_ethtool_rule *eth_rule)
 510{
 511	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
 512	if (eth_rule->rule)
 513		mlx5_del_flow_rules(eth_rule->rule);
 514	if (eth_rule->rss)
 515		mlx5e_rss_refcnt_dec(eth_rule->rss);
 516	list_del(&eth_rule->list);
 517	ethtool->tot_num_rules--;
 518	put_flow_table(eth_rule->eth_ft);
 519	kfree(eth_rule);
 520}
 521
 522static struct mlx5e_ethtool_rule *find_ethtool_rule(struct mlx5e_priv *priv,
 523						    int location)
 524{
 525	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 526	struct mlx5e_ethtool_rule *iter;
 527
 528	list_for_each_entry(iter, &ethtool->rules, list) {
 529		if (iter->flow_spec.location == location)
 530			return iter;
 531	}
 532	return NULL;
 533}
 534
 535static struct mlx5e_ethtool_rule *get_ethtool_rule(struct mlx5e_priv *priv,
 536						   int location)
 537{
 538	struct mlx5e_ethtool_rule *eth_rule;
 539
 540	eth_rule = find_ethtool_rule(priv, location);
 541	if (eth_rule)
 542		del_ethtool_rule(priv->fs, eth_rule);
 543
 544	eth_rule = kzalloc(sizeof(*eth_rule), GFP_KERNEL);
 545	if (!eth_rule)
 546		return ERR_PTR(-ENOMEM);
 547
 548	add_rule_to_list(priv, eth_rule);
 549	return eth_rule;
 550}
 551
 552#define MAX_NUM_OF_ETHTOOL_RULES BIT(10)
 553
 554#define all_ones(field) (field == (__force typeof(field))-1)
 555#define all_zeros_or_all_ones(field)		\
 556	((field) == 0 || (field) == (__force typeof(field))-1)
 557
 558static int validate_ethter(struct ethtool_rx_flow_spec *fs)
 559{
 560	struct ethhdr *eth_mask = &fs->m_u.ether_spec;
 561	int ntuples = 0;
 562
 563	if (!is_zero_ether_addr(eth_mask->h_dest))
 564		ntuples++;
 565	if (!is_zero_ether_addr(eth_mask->h_source))
 566		ntuples++;
 567	if (eth_mask->h_proto)
 568		ntuples++;
 569	return ntuples;
 570}
 571
 572static int validate_tcpudp4(struct ethtool_rx_flow_spec *fs)
 573{
 574	struct ethtool_tcpip4_spec *l4_mask = &fs->m_u.tcp_ip4_spec;
 575	int ntuples = 0;
 576
 577	if (l4_mask->tos)
 578		return -EINVAL;
 579
 580	if (l4_mask->ip4src)
 581		ntuples++;
 582	if (l4_mask->ip4dst)
 583		ntuples++;
 584	if (l4_mask->psrc)
 585		ntuples++;
 586	if (l4_mask->pdst)
 587		ntuples++;
 588	/* Flow is TCP/UDP */
 589	return ++ntuples;
 590}
 591
 592static int validate_ip4(struct ethtool_rx_flow_spec *fs)
 593{
 594	struct ethtool_usrip4_spec *l3_mask = &fs->m_u.usr_ip4_spec;
 595	int ntuples = 0;
 596
 597	if (l3_mask->l4_4_bytes || l3_mask->tos ||
 598	    fs->h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4)
 599		return -EINVAL;
 600	if (l3_mask->ip4src)
 601		ntuples++;
 602	if (l3_mask->ip4dst)
 603		ntuples++;
 604	if (l3_mask->proto)
 605		ntuples++;
 606	/* Flow is IPv4 */
 607	return ++ntuples;
 608}
 609
 610static int validate_ip6(struct ethtool_rx_flow_spec *fs)
 611{
 612	struct ethtool_usrip6_spec *l3_mask = &fs->m_u.usr_ip6_spec;
 613	int ntuples = 0;
 614
 615	if (l3_mask->l4_4_bytes || l3_mask->tclass)
 616		return -EINVAL;
 617	if (!ipv6_addr_any((struct in6_addr *)l3_mask->ip6src))
 618		ntuples++;
 619
 620	if (!ipv6_addr_any((struct in6_addr *)l3_mask->ip6dst))
 621		ntuples++;
 622	if (l3_mask->l4_proto)
 623		ntuples++;
 624	/* Flow is IPv6 */
 625	return ++ntuples;
 626}
 627
 628static int validate_tcpudp6(struct ethtool_rx_flow_spec *fs)
 629{
 630	struct ethtool_tcpip6_spec *l4_mask = &fs->m_u.tcp_ip6_spec;
 631	int ntuples = 0;
 632
 633	if (l4_mask->tclass)
 634		return -EINVAL;
 635
 636	if (!ipv6_addr_any((struct in6_addr *)l4_mask->ip6src))
 637		ntuples++;
 638
 639	if (!ipv6_addr_any((struct in6_addr *)l4_mask->ip6dst))
 640		ntuples++;
 641
 642	if (l4_mask->psrc)
 643		ntuples++;
 644	if (l4_mask->pdst)
 645		ntuples++;
 646	/* Flow is TCP/UDP */
 647	return ++ntuples;
 648}
 649
 650static int validate_vlan(struct ethtool_rx_flow_spec *fs)
 651{
 652	if (fs->m_ext.vlan_etype ||
 653	    fs->m_ext.vlan_tci != cpu_to_be16(VLAN_VID_MASK))
 654		return -EINVAL;
 655
 656	if (fs->m_ext.vlan_tci &&
 657	    (be16_to_cpu(fs->h_ext.vlan_tci) >= VLAN_N_VID))
 658		return -EINVAL;
 659
 660	return 1;
 661}
 662
 663static int validate_flow(struct mlx5e_priv *priv,
 664			 struct ethtool_rx_flow_spec *fs)
 665{
 666	int num_tuples = 0;
 667	int ret = 0;
 668
 669	if (fs->location >= MAX_NUM_OF_ETHTOOL_RULES)
 670		return -ENOSPC;
 671
 672	if (fs->ring_cookie != RX_CLS_FLOW_DISC)
 673		if (fs->ring_cookie >= priv->channels.params.num_channels)
 674			return -EINVAL;
 675
 676	switch (flow_type_mask(fs->flow_type)) {
 677	case ETHER_FLOW:
 678		num_tuples += validate_ethter(fs);
 679		break;
 680	case TCP_V4_FLOW:
 681	case UDP_V4_FLOW:
 682		ret = validate_tcpudp4(fs);
 683		if (ret < 0)
 684			return ret;
 685		num_tuples += ret;
 686		break;
 687	case IP_USER_FLOW:
 688		ret = validate_ip4(fs);
 689		if (ret < 0)
 690			return ret;
 691		num_tuples += ret;
 692		break;
 693	case TCP_V6_FLOW:
 694	case UDP_V6_FLOW:
 695		ret = validate_tcpudp6(fs);
 696		if (ret < 0)
 697			return ret;
 698		num_tuples += ret;
 699		break;
 700	case IPV6_USER_FLOW:
 701		ret = validate_ip6(fs);
 702		if (ret < 0)
 703			return ret;
 704		num_tuples += ret;
 705		break;
 706	default:
 707		return -EOPNOTSUPP;
 708	}
 709	if ((fs->flow_type & FLOW_EXT)) {
 710		ret = validate_vlan(fs);
 711		if (ret < 0)
 712			return ret;
 713		num_tuples += ret;
 714	}
 715
 716	if (fs->flow_type & FLOW_MAC_EXT &&
 717	    !is_zero_ether_addr(fs->m_ext.h_dest))
 718		num_tuples++;
 719
 720	return num_tuples;
 721}
 722
 723static int
 724mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv,
 725			   struct ethtool_rx_flow_spec *fs, u32 rss_context)
 726{
 727	struct mlx5e_ethtool_table *eth_ft;
 728	struct mlx5e_ethtool_rule *eth_rule;
 729	struct mlx5_flow_handle *rule;
 730	int num_tuples;
 731	int err;
 732
 733	num_tuples = validate_flow(priv, fs);
 734	if (num_tuples <= 0) {
 735		netdev_warn(priv->netdev, "%s: flow is not valid %d\n",
 736			    __func__, num_tuples);
 737		return num_tuples < 0 ? num_tuples : -EINVAL;
 738	}
 739
 740	eth_ft = get_flow_table(priv, fs, num_tuples);
 741	if (IS_ERR(eth_ft))
 742		return PTR_ERR(eth_ft);
 743
 744	eth_rule = get_ethtool_rule(priv, fs->location);
 745	if (IS_ERR(eth_rule)) {
 746		put_flow_table(eth_ft);
 747		return PTR_ERR(eth_rule);
 748	}
 749
 750	eth_rule->flow_spec = *fs;
 751	eth_rule->eth_ft = eth_ft;
 752
 753	rule = add_ethtool_flow_rule(priv, eth_rule, eth_ft->ft, fs, rss_context);
 754	if (IS_ERR(rule)) {
 755		err = PTR_ERR(rule);
 756		goto del_ethtool_rule;
 757	}
 758
 759	eth_rule->rule = rule;
 760
 761	return 0;
 762
 763del_ethtool_rule:
 764	del_ethtool_rule(priv->fs, eth_rule);
 765
 766	return err;
 767}
 768
 769static int
 770mlx5e_ethtool_flow_remove(struct mlx5e_priv *priv, int location)
 771{
 772	struct mlx5e_ethtool_rule *eth_rule;
 773	int err = 0;
 774
 775	if (location >= MAX_NUM_OF_ETHTOOL_RULES)
 776		return -ENOSPC;
 777
 778	eth_rule = find_ethtool_rule(priv, location);
 779	if (!eth_rule) {
 780		err =  -ENOENT;
 781		goto out;
 782	}
 783
 784	del_ethtool_rule(priv->fs, eth_rule);
 785out:
 786	return err;
 787}
 788
 789static int
 790mlx5e_ethtool_get_flow(struct mlx5e_priv *priv,
 791		       struct ethtool_rxnfc *info, int location)
 792{
 793	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 794	struct mlx5e_ethtool_rule *eth_rule;
 795
 796	if (location < 0 || location >= MAX_NUM_OF_ETHTOOL_RULES)
 797		return -EINVAL;
 798
 799	list_for_each_entry(eth_rule, &ethtool->rules, list) {
 800		int index;
 801
 802		if (eth_rule->flow_spec.location != location)
 803			continue;
 804		if (!info)
 805			return 0;
 806		info->fs = eth_rule->flow_spec;
 807		if (!eth_rule->rss)
 808			return 0;
 809		index = mlx5e_rx_res_rss_index(priv->rx_res, eth_rule->rss);
 810		if (index < 0)
 811			return index;
 812		info->rss_context = index;
 813		return 0;
 814	}
 815
 816	return -ENOENT;
 817}
 818
 819static int
 820mlx5e_ethtool_get_all_flows(struct mlx5e_priv *priv,
 821			    struct ethtool_rxnfc *info, u32 *rule_locs)
 822{
 823	int location = 0;
 824	int idx = 0;
 825	int err = 0;
 826
 827	info->data = MAX_NUM_OF_ETHTOOL_RULES;
 828	while ((!err || err == -ENOENT) && idx < info->rule_cnt) {
 829		err = mlx5e_ethtool_get_flow(priv, NULL, location);
 830		if (!err)
 831			rule_locs[idx++] = location;
 832		location++;
 833	}
 834	return err;
 835}
 836
 837int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool)
 838{
 839	*ethtool =  kvzalloc(sizeof(**ethtool), GFP_KERNEL);
 840	if (!*ethtool)
 841		return -ENOMEM;
 842	return 0;
 843}
 844
 845void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool)
 846{
 847	kvfree(ethtool);
 848}
 849
 850void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs)
 851{
 852	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
 853	struct mlx5e_ethtool_rule *iter;
 854	struct mlx5e_ethtool_rule *temp;
 855
 856	list_for_each_entry_safe(iter, temp, &ethtool->rules, list)
 857		del_ethtool_rule(fs, iter);
 858}
 859
 860void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs)
 861{
 862	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
 863
 864	INIT_LIST_HEAD(&ethtool->rules);
 865}
 866
 867static int flow_type_to_traffic_type(u32 flow_type)
 868{
 869	switch (flow_type) {
 870	case TCP_V4_FLOW:
 871		return MLX5_TT_IPV4_TCP;
 872	case TCP_V6_FLOW:
 873		return MLX5_TT_IPV6_TCP;
 874	case UDP_V4_FLOW:
 875		return MLX5_TT_IPV4_UDP;
 876	case UDP_V6_FLOW:
 877		return MLX5_TT_IPV6_UDP;
 878	case AH_V4_FLOW:
 879		return MLX5_TT_IPV4_IPSEC_AH;
 880	case AH_V6_FLOW:
 881		return MLX5_TT_IPV6_IPSEC_AH;
 882	case ESP_V4_FLOW:
 883		return MLX5_TT_IPV4_IPSEC_ESP;
 884	case ESP_V6_FLOW:
 885		return MLX5_TT_IPV6_IPSEC_ESP;
 886	case IPV4_FLOW:
 887	case IP_USER_FLOW:
 888		return MLX5_TT_IPV4;
 889	case IPV6_FLOW:
 890	case IPV6_USER_FLOW:
 891		return MLX5_TT_IPV6;
 892	default:
 893		return -EINVAL;
 894	}
 895}
 896
 897int mlx5e_ethtool_set_rxfh_fields(struct mlx5e_priv *priv,
 898				  const struct ethtool_rxfh_fields *nfc,
 899				  struct netlink_ext_ack *extack)
 900{
 901	u8 rx_hash_field = 0;
 902	u32 flow_type = 0;
 903	u32 rss_idx;
 904	int err;
 905	int tt;
 906
 907	rss_idx = nfc->rss_context;
 908
 909	flow_type = flow_type_mask(nfc->flow_type);
 910	tt = flow_type_to_traffic_type(flow_type);
 911	if (tt < 0)
 912		return tt;
 913
 914	/*  RSS does not support anything other than hashing to queues
 915	 *  on src IP, dest IP, TCP/UDP src port and TCP/UDP dest
 916	 *  port.
 917	 */
 918	if (flow_type != TCP_V4_FLOW &&
 919	    flow_type != TCP_V6_FLOW &&
 920	    flow_type != UDP_V4_FLOW &&
 921	    flow_type != UDP_V6_FLOW)
 922		return -EOPNOTSUPP;
 923
 924	if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST |
 925			  RXH_L4_B_0_1 | RXH_L4_B_2_3))
 926		return -EOPNOTSUPP;
 927
 928	if (nfc->data & RXH_IP_SRC)
 929		rx_hash_field |= MLX5_HASH_FIELD_SEL_SRC_IP;
 930	if (nfc->data & RXH_IP_DST)
 931		rx_hash_field |= MLX5_HASH_FIELD_SEL_DST_IP;
 932	if (nfc->data & RXH_L4_B_0_1)
 933		rx_hash_field |= MLX5_HASH_FIELD_SEL_L4_SPORT;
 934	if (nfc->data & RXH_L4_B_2_3)
 935		rx_hash_field |= MLX5_HASH_FIELD_SEL_L4_DPORT;
 936
 937	mutex_lock(&priv->state_lock);
 938	err = mlx5e_rx_res_rss_set_hash_fields(priv->rx_res, rss_idx, tt, rx_hash_field);
 939	mutex_unlock(&priv->state_lock);
 940
 941	return err;
 942}
 943
 944int mlx5e_ethtool_get_rxfh_fields(struct mlx5e_priv *priv,
 945				  struct ethtool_rxfh_fields *nfc)
 946{
 947	int hash_field = 0;
 948	u32 flow_type = 0;
 949	u32 rss_idx;
 950	int tt;
 951
 952	rss_idx = nfc->rss_context;
 953
 954	flow_type = flow_type_mask(nfc->flow_type);
 955	tt = flow_type_to_traffic_type(flow_type);
 956	if (tt < 0)
 957		return tt;
 958
 959	hash_field = mlx5e_rx_res_rss_get_hash_fields(priv->rx_res, rss_idx, tt);
 960	if (hash_field < 0)
 961		return hash_field;
 962
 963	nfc->data = 0;
 964
 965	if (hash_field & MLX5_HASH_FIELD_SEL_SRC_IP)
 966		nfc->data |= RXH_IP_SRC;
 967	if (hash_field & MLX5_HASH_FIELD_SEL_DST_IP)
 968		nfc->data |= RXH_IP_DST;
 969	if (hash_field & MLX5_HASH_FIELD_SEL_L4_SPORT)
 970		nfc->data |= RXH_L4_B_0_1;
 971	if (hash_field & MLX5_HASH_FIELD_SEL_L4_DPORT)
 972		nfc->data |= RXH_L4_B_2_3;
 973
 974	return 0;
 975}
 976
 977int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd)
 978{
 979	int err = 0;
 980
 981	switch (cmd->cmd) {
 982	case ETHTOOL_SRXCLSRLINS:
 983		err = mlx5e_ethtool_flow_replace(priv, &cmd->fs, cmd->rss_context);
 984		break;
 985	case ETHTOOL_SRXCLSRLDEL:
 986		err = mlx5e_ethtool_flow_remove(priv, cmd->fs.location);
 987		break;
 988	default:
 989		err = -EOPNOTSUPP;
 990		break;
 991	}
 992
 993	return err;
 994}
 995
 996int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
 997			    struct ethtool_rxnfc *info, u32 *rule_locs)
 998{
 999	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
1000	int err = 0;
1001
1002	switch (info->cmd) {
1003	case ETHTOOL_GRXCLSRLCNT:
1004		info->rule_cnt = ethtool->tot_num_rules;
1005		break;
1006	case ETHTOOL_GRXCLSRULE:
1007		err = mlx5e_ethtool_get_flow(priv, info, info->fs.location);
1008		break;
1009	case ETHTOOL_GRXCLSRLALL:
1010		err = mlx5e_ethtool_get_all_flows(priv, info, rule_locs);
1011		break;
1012	default:
1013		err = -EOPNOTSUPP;
1014		break;
1015	}
1016
1017	return err;
1018}
1019