fs.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4 */
   5
   6#include <rdma/ib_user_verbs.h>
   7#include <rdma/ib_verbs.h>
   8#include <rdma/uverbs_types.h>
   9#include <rdma/uverbs_ioctl.h>
  10#include <rdma/uverbs_std_types.h>
  11#include <rdma/mlx5_user_ioctl_cmds.h>
  12#include <rdma/mlx5_user_ioctl_verbs.h>
  13#include <rdma/ib_hdrs.h>
  14#include <rdma/ib_umem.h>
  15#include <rdma/ib_ucaps.h>
  16#include <linux/mlx5/driver.h>
  17#include <linux/mlx5/fs.h>
  18#include <linux/mlx5/fs_helpers.h>
  19#include <linux/mlx5/eswitch.h>
  20#include <net/inet_ecn.h>
  21#include "mlx5_ib.h"
  22#include "counters.h"
  23#include "devx.h"
  24#include "fs.h"
  25
  26#define UVERBS_MODULE_NAME mlx5_ib
  27#include <rdma/uverbs_named_ioctl.h>
  28
  29enum {
  30	MATCH_CRITERIA_ENABLE_OUTER_BIT,
  31	MATCH_CRITERIA_ENABLE_MISC_BIT,
  32	MATCH_CRITERIA_ENABLE_INNER_BIT,
  33	MATCH_CRITERIA_ENABLE_MISC2_BIT
  34};
  35
  36
  37struct mlx5_per_qp_opfc {
  38	struct mlx5_ib_op_fc opfcs[MLX5_IB_OPCOUNTER_MAX];
  39};
  40
  41#define HEADER_IS_ZERO(match_criteria, headers)			           \
  42	!(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
  43		    0, MLX5_FLD_SZ_BYTES(fte_match_param, headers)))       \
  44
  45static u8 get_match_criteria_enable(u32 *match_criteria)
  46{
  47	u8 match_criteria_enable;
  48
  49	match_criteria_enable =
  50		(!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
  51		MATCH_CRITERIA_ENABLE_OUTER_BIT;
  52	match_criteria_enable |=
  53		(!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
  54		MATCH_CRITERIA_ENABLE_MISC_BIT;
  55	match_criteria_enable |=
  56		(!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
  57		MATCH_CRITERIA_ENABLE_INNER_BIT;
  58	match_criteria_enable |=
  59		(!HEADER_IS_ZERO(match_criteria, misc_parameters_2)) <<
  60		MATCH_CRITERIA_ENABLE_MISC2_BIT;
  61
  62	return match_criteria_enable;
  63}
  64
  65static int set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
  66{
  67	u8 entry_mask;
  68	u8 entry_val;
  69	int err = 0;
  70
  71	if (!mask)
  72		goto out;
  73
  74	entry_mask = MLX5_GET(fte_match_set_lyr_2_4, outer_c,
  75			      ip_protocol);
  76	entry_val = MLX5_GET(fte_match_set_lyr_2_4, outer_v,
  77			     ip_protocol);
  78	if (!entry_mask) {
  79		MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
  80		MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
  81		goto out;
  82	}
  83	/* Don't override existing ip protocol */
  84	if (mask != entry_mask || val != entry_val)
  85		err = -EINVAL;
  86out:
  87	return err;
  88}
  89
  90static void set_flow_label(void *misc_c, void *misc_v, u32 mask, u32 val,
  91			   bool inner)
  92{
  93	if (inner) {
  94		MLX5_SET(fte_match_set_misc,
  95			 misc_c, inner_ipv6_flow_label, mask);
  96		MLX5_SET(fte_match_set_misc,
  97			 misc_v, inner_ipv6_flow_label, val);
  98	} else {
  99		MLX5_SET(fte_match_set_misc,
 100			 misc_c, outer_ipv6_flow_label, mask);
 101		MLX5_SET(fte_match_set_misc,
 102			 misc_v, outer_ipv6_flow_label, val);
 103	}
 104}
 105
 106static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
 107{
 108	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
 109	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
 110	MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
 111	MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
 112}
 113
 114static int check_mpls_supp_fields(u32 field_support, const __be32 *set_mask)
 115{
 116	if (MLX5_GET(fte_match_mpls, set_mask, mpls_label) &&
 117	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_LABEL))
 118		return -EOPNOTSUPP;
 119
 120	if (MLX5_GET(fte_match_mpls, set_mask, mpls_exp) &&
 121	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_EXP))
 122		return -EOPNOTSUPP;
 123
 124	if (MLX5_GET(fte_match_mpls, set_mask, mpls_s_bos) &&
 125	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_S_BOS))
 126		return -EOPNOTSUPP;
 127
 128	if (MLX5_GET(fte_match_mpls, set_mask, mpls_ttl) &&
 129	    !(field_support & MLX5_FIELD_SUPPORT_MPLS_TTL))
 130		return -EOPNOTSUPP;
 131
 132	return 0;
 133}
 134
 135#define LAST_ETH_FIELD vlan_tag
 136#define LAST_IPV4_FIELD tos
 137#define LAST_IPV6_FIELD traffic_class
 138#define LAST_TCP_UDP_FIELD src_port
 139#define LAST_TUNNEL_FIELD tunnel_id
 140#define LAST_FLOW_TAG_FIELD tag_id
 141#define LAST_DROP_FIELD size
 142#define LAST_COUNTERS_FIELD counters
 143
 144/* Field is the last supported field */
 145#define FIELDS_NOT_SUPPORTED(filter, field)                                    \
 146	memchr_inv((void *)&filter.field + sizeof(filter.field), 0,            \
 147		   sizeof(filter) - offsetofend(typeof(filter), field))
 148
 149int parse_flow_flow_action(struct mlx5_ib_flow_action *maction,
 150			   bool is_egress,
 151			   struct mlx5_flow_act *action)
 152{
 153
 154	switch (maction->ib_action.type) {
 155	case IB_FLOW_ACTION_UNSPECIFIED:
 156		if (maction->flow_action_raw.sub_type ==
 157		    MLX5_IB_FLOW_ACTION_MODIFY_HEADER) {
 158			if (action->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
 159				return -EINVAL;
 160			action->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
 161			action->modify_hdr =
 162				maction->flow_action_raw.modify_hdr;
 163			return 0;
 164		}
 165		if (maction->flow_action_raw.sub_type ==
 166		    MLX5_IB_FLOW_ACTION_DECAP) {
 167			if (action->action & MLX5_FLOW_CONTEXT_ACTION_DECAP)
 168				return -EINVAL;
 169			action->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
 170			return 0;
 171		}
 172		if (maction->flow_action_raw.sub_type ==
 173		    MLX5_IB_FLOW_ACTION_PACKET_REFORMAT) {
 174			if (action->action &
 175			    MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT)
 176				return -EINVAL;
 177			action->action |=
 178				MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 179			action->pkt_reformat =
 180				maction->flow_action_raw.pkt_reformat;
 181			return 0;
 182		}
 183		fallthrough;
 184	default:
 185		return -EOPNOTSUPP;
 186	}
 187}
 188
 189static int parse_flow_attr(struct mlx5_core_dev *mdev,
 190			   struct mlx5_flow_spec *spec,
 191			   const union ib_flow_spec *ib_spec,
 192			   const struct ib_flow_attr *flow_attr,
 193			   struct mlx5_flow_act *action, u32 prev_type)
 194{
 195	struct mlx5_flow_context *flow_context = &spec->flow_context;
 196	u32 *match_c = spec->match_criteria;
 197	u32 *match_v = spec->match_value;
 198	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
 199					   misc_parameters);
 200	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
 201					   misc_parameters);
 202	void *misc_params2_c = MLX5_ADDR_OF(fte_match_param, match_c,
 203					    misc_parameters_2);
 204	void *misc_params2_v = MLX5_ADDR_OF(fte_match_param, match_v,
 205					    misc_parameters_2);
 206	void *headers_c;
 207	void *headers_v;
 208	int match_ipv;
 209	int ret;
 210
 211	if (ib_spec->type & IB_FLOW_SPEC_INNER) {
 212		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
 213					 inner_headers);
 214		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
 215					 inner_headers);
 216		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 217					ft_field_support.inner_ip_version);
 218	} else {
 219		headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
 220					 outer_headers);
 221		headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
 222					 outer_headers);
 223		match_ipv = MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 224					ft_field_support.outer_ip_version);
 225	}
 226
 227	switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) {
 228	case IB_FLOW_SPEC_ETH:
 229		if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
 230			return -EOPNOTSUPP;
 231
 232		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 233					     dmac_47_16),
 234				ib_spec->eth.mask.dst_mac);
 235		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 236					     dmac_47_16),
 237				ib_spec->eth.val.dst_mac);
 238
 239		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 240					     smac_47_16),
 241				ib_spec->eth.mask.src_mac);
 242		ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 243					     smac_47_16),
 244				ib_spec->eth.val.src_mac);
 245
 246		if (ib_spec->eth.mask.vlan_tag) {
 247			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 248				 cvlan_tag, 1);
 249			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 250				 cvlan_tag, 1);
 251
 252			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 253				 first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
 254			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 255				 first_vid, ntohs(ib_spec->eth.val.vlan_tag));
 256
 257			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 258				 first_cfi,
 259				 ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
 260			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 261				 first_cfi,
 262				 ntohs(ib_spec->eth.val.vlan_tag) >> 12);
 263
 264			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 265				 first_prio,
 266				 ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
 267			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 268				 first_prio,
 269				 ntohs(ib_spec->eth.val.vlan_tag) >> 13);
 270		}
 271		MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 272			 ethertype, ntohs(ib_spec->eth.mask.ether_type));
 273		MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 274			 ethertype, ntohs(ib_spec->eth.val.ether_type));
 275		break;
 276	case IB_FLOW_SPEC_IPV4:
 277		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
 278			return -EOPNOTSUPP;
 279
 280		if (match_ipv) {
 281			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 282				 ip_version, 0xf);
 283			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 284				 ip_version, MLX5_FS_IPV4_VERSION);
 285		} else {
 286			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 287				 ethertype, 0xffff);
 288			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 289				 ethertype, ETH_P_IP);
 290		}
 291
 292		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 293				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 294		       &ib_spec->ipv4.mask.src_ip,
 295		       sizeof(ib_spec->ipv4.mask.src_ip));
 296		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 297				    src_ipv4_src_ipv6.ipv4_layout.ipv4),
 298		       &ib_spec->ipv4.val.src_ip,
 299		       sizeof(ib_spec->ipv4.val.src_ip));
 300		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 301				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 302		       &ib_spec->ipv4.mask.dst_ip,
 303		       sizeof(ib_spec->ipv4.mask.dst_ip));
 304		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 305				    dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
 306		       &ib_spec->ipv4.val.dst_ip,
 307		       sizeof(ib_spec->ipv4.val.dst_ip));
 308
 309		set_tos(headers_c, headers_v,
 310			ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
 311
 312		if (set_proto(headers_c, headers_v,
 313			      ib_spec->ipv4.mask.proto,
 314			      ib_spec->ipv4.val.proto))
 315			return -EINVAL;
 316		break;
 317	case IB_FLOW_SPEC_IPV6:
 318		if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
 319			return -EOPNOTSUPP;
 320
 321		if (match_ipv) {
 322			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 323				 ip_version, 0xf);
 324			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 325				 ip_version, MLX5_FS_IPV6_VERSION);
 326		} else {
 327			MLX5_SET(fte_match_set_lyr_2_4, headers_c,
 328				 ethertype, 0xffff);
 329			MLX5_SET(fte_match_set_lyr_2_4, headers_v,
 330				 ethertype, ETH_P_IPV6);
 331		}
 332
 333		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 334				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 335		       &ib_spec->ipv6.mask.src_ip,
 336		       sizeof(ib_spec->ipv6.mask.src_ip));
 337		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 338				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
 339		       &ib_spec->ipv6.val.src_ip,
 340		       sizeof(ib_spec->ipv6.val.src_ip));
 341		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
 342				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 343		       &ib_spec->ipv6.mask.dst_ip,
 344		       sizeof(ib_spec->ipv6.mask.dst_ip));
 345		memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
 346				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 347		       &ib_spec->ipv6.val.dst_ip,
 348		       sizeof(ib_spec->ipv6.val.dst_ip));
 349
 350		set_tos(headers_c, headers_v,
 351			ib_spec->ipv6.mask.traffic_class,
 352			ib_spec->ipv6.val.traffic_class);
 353
 354		if (set_proto(headers_c, headers_v,
 355			      ib_spec->ipv6.mask.next_hdr,
 356			      ib_spec->ipv6.val.next_hdr))
 357			return -EINVAL;
 358
 359		set_flow_label(misc_params_c, misc_params_v,
 360			       ntohl(ib_spec->ipv6.mask.flow_label),
 361			       ntohl(ib_spec->ipv6.val.flow_label),
 362			       ib_spec->type & IB_FLOW_SPEC_INNER);
 363		break;
 364	case IB_FLOW_SPEC_ESP:
 365		return -EOPNOTSUPP;
 366	case IB_FLOW_SPEC_TCP:
 367		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
 368					 LAST_TCP_UDP_FIELD))
 369			return -EOPNOTSUPP;
 370
 371		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_TCP))
 372			return -EINVAL;
 373
 374		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_sport,
 375			 ntohs(ib_spec->tcp_udp.mask.src_port));
 376		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_sport,
 377			 ntohs(ib_spec->tcp_udp.val.src_port));
 378
 379		MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_dport,
 380			 ntohs(ib_spec->tcp_udp.mask.dst_port));
 381		MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_dport,
 382			 ntohs(ib_spec->tcp_udp.val.dst_port));
 383		break;
 384	case IB_FLOW_SPEC_UDP:
 385		if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
 386					 LAST_TCP_UDP_FIELD))
 387			return -EOPNOTSUPP;
 388
 389		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_UDP))
 390			return -EINVAL;
 391
 392		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
 393			 ntohs(ib_spec->tcp_udp.mask.src_port));
 394		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
 395			 ntohs(ib_spec->tcp_udp.val.src_port));
 396
 397		MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
 398			 ntohs(ib_spec->tcp_udp.mask.dst_port));
 399		MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
 400			 ntohs(ib_spec->tcp_udp.val.dst_port));
 401		break;
 402	case IB_FLOW_SPEC_GRE:
 403		if (ib_spec->gre.mask.c_ks_res0_ver)
 404			return -EOPNOTSUPP;
 405
 406		if (set_proto(headers_c, headers_v, 0xff, IPPROTO_GRE))
 407			return -EINVAL;
 408
 409		MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
 410			 0xff);
 411		MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 412			 IPPROTO_GRE);
 413
 414		MLX5_SET(fte_match_set_misc, misc_params_c, gre_protocol,
 415			 ntohs(ib_spec->gre.mask.protocol));
 416		MLX5_SET(fte_match_set_misc, misc_params_v, gre_protocol,
 417			 ntohs(ib_spec->gre.val.protocol));
 418
 419		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_c,
 420				    gre_key.nvgre.hi),
 421		       &ib_spec->gre.mask.key,
 422		       sizeof(ib_spec->gre.mask.key));
 423		memcpy(MLX5_ADDR_OF(fte_match_set_misc, misc_params_v,
 424				    gre_key.nvgre.hi),
 425		       &ib_spec->gre.val.key,
 426		       sizeof(ib_spec->gre.val.key));
 427		break;
 428	case IB_FLOW_SPEC_MPLS:
 429		switch (prev_type) {
 430		case IB_FLOW_SPEC_UDP:
 431			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 432						   ft_field_support.outer_first_mpls_over_udp),
 433						   &ib_spec->mpls.mask.tag))
 434				return -EOPNOTSUPP;
 435
 436			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
 437					    outer_first_mpls_over_udp),
 438			       &ib_spec->mpls.val.tag,
 439			       sizeof(ib_spec->mpls.val.tag));
 440			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
 441					    outer_first_mpls_over_udp),
 442			       &ib_spec->mpls.mask.tag,
 443			       sizeof(ib_spec->mpls.mask.tag));
 444			break;
 445		case IB_FLOW_SPEC_GRE:
 446			if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 447						   ft_field_support.outer_first_mpls_over_gre),
 448						   &ib_spec->mpls.mask.tag))
 449				return -EOPNOTSUPP;
 450
 451			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
 452					    outer_first_mpls_over_gre),
 453			       &ib_spec->mpls.val.tag,
 454			       sizeof(ib_spec->mpls.val.tag));
 455			memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
 456					    outer_first_mpls_over_gre),
 457			       &ib_spec->mpls.mask.tag,
 458			       sizeof(ib_spec->mpls.mask.tag));
 459			break;
 460		default:
 461			if (ib_spec->type & IB_FLOW_SPEC_INNER) {
 462				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 463							   ft_field_support.inner_first_mpls),
 464							   &ib_spec->mpls.mask.tag))
 465					return -EOPNOTSUPP;
 466
 467				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
 468						    inner_first_mpls),
 469				       &ib_spec->mpls.val.tag,
 470				       sizeof(ib_spec->mpls.val.tag));
 471				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
 472						    inner_first_mpls),
 473				       &ib_spec->mpls.mask.tag,
 474				       sizeof(ib_spec->mpls.mask.tag));
 475			} else {
 476				if (check_mpls_supp_fields(MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 477							   ft_field_support.outer_first_mpls),
 478							   &ib_spec->mpls.mask.tag))
 479					return -EOPNOTSUPP;
 480
 481				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_v,
 482						    outer_first_mpls),
 483				       &ib_spec->mpls.val.tag,
 484				       sizeof(ib_spec->mpls.val.tag));
 485				memcpy(MLX5_ADDR_OF(fte_match_set_misc2, misc_params2_c,
 486						    outer_first_mpls),
 487				       &ib_spec->mpls.mask.tag,
 488				       sizeof(ib_spec->mpls.mask.tag));
 489			}
 490		}
 491		break;
 492	case IB_FLOW_SPEC_VXLAN_TUNNEL:
 493		if (FIELDS_NOT_SUPPORTED(ib_spec->tunnel.mask,
 494					 LAST_TUNNEL_FIELD))
 495			return -EOPNOTSUPP;
 496
 497		MLX5_SET(fte_match_set_misc, misc_params_c, vxlan_vni,
 498			 ntohl(ib_spec->tunnel.mask.tunnel_id));
 499		MLX5_SET(fte_match_set_misc, misc_params_v, vxlan_vni,
 500			 ntohl(ib_spec->tunnel.val.tunnel_id));
 501		break;
 502	case IB_FLOW_SPEC_ACTION_TAG:
 503		if (FIELDS_NOT_SUPPORTED(ib_spec->flow_tag,
 504					 LAST_FLOW_TAG_FIELD))
 505			return -EOPNOTSUPP;
 506		if (ib_spec->flow_tag.tag_id >= BIT(24))
 507			return -EINVAL;
 508
 509		flow_context->flow_tag = ib_spec->flow_tag.tag_id;
 510		flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
 511		break;
 512	case IB_FLOW_SPEC_ACTION_DROP:
 513		if (FIELDS_NOT_SUPPORTED(ib_spec->drop,
 514					 LAST_DROP_FIELD))
 515			return -EOPNOTSUPP;
 516		action->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
 517		break;
 518	case IB_FLOW_SPEC_ACTION_HANDLE:
 519		ret = parse_flow_flow_action(to_mflow_act(ib_spec->action.act),
 520			flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS, action);
 521		if (ret)
 522			return ret;
 523		break;
 524	case IB_FLOW_SPEC_ACTION_COUNT:
 525		if (FIELDS_NOT_SUPPORTED(ib_spec->flow_count,
 526					 LAST_COUNTERS_FIELD))
 527			return -EOPNOTSUPP;
 528
 529		/* for now support only one counters spec per flow */
 530		if (action->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
 531			return -EINVAL;
 532
 533		action->counters = ib_spec->flow_count.counters;
 534		action->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
 535		break;
 536	default:
 537		return -EINVAL;
 538	}
 539
 540	return 0;
 541}
 542
 543/* If a flow could catch both multicast and unicast packets,
 544 * it won't fall into the multicast flow steering table and this rule
 545 * could steal other multicast packets.
 546 */
 547static bool flow_is_multicast_only(const struct ib_flow_attr *ib_attr)
 548{
 549	union ib_flow_spec *flow_spec;
 550
 551	if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
 552	    ib_attr->num_of_specs < 1)
 553		return false;
 554
 555	flow_spec = (union ib_flow_spec *)(ib_attr + 1);
 556	if (flow_spec->type == IB_FLOW_SPEC_IPV4) {
 557		struct ib_flow_spec_ipv4 *ipv4_spec;
 558
 559		ipv4_spec = (struct ib_flow_spec_ipv4 *)flow_spec;
 560		if (ipv4_is_multicast(ipv4_spec->val.dst_ip))
 561			return true;
 562
 563		return false;
 564	}
 565
 566	if (flow_spec->type == IB_FLOW_SPEC_ETH) {
 567		struct ib_flow_spec_eth *eth_spec;
 568
 569		eth_spec = (struct ib_flow_spec_eth *)flow_spec;
 570		return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
 571		       is_multicast_ether_addr(eth_spec->val.dst_mac);
 572	}
 573
 574	return false;
 575}
 576
 577static bool is_valid_ethertype(struct mlx5_core_dev *mdev,
 578			       const struct ib_flow_attr *flow_attr,
 579			       bool check_inner)
 580{
 581	union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
 582	int match_ipv = check_inner ?
 583			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 584					ft_field_support.inner_ip_version) :
 585			MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 586					ft_field_support.outer_ip_version);
 587	int inner_bit = check_inner ? IB_FLOW_SPEC_INNER : 0;
 588	bool ipv4_spec_valid, ipv6_spec_valid;
 589	unsigned int ip_spec_type = 0;
 590	bool has_ethertype = false;
 591	unsigned int spec_index;
 592	bool mask_valid = true;
 593	u16 eth_type = 0;
 594	bool type_valid;
 595
 596	/* Validate that ethertype is correct */
 597	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
 598		if ((ib_spec->type == (IB_FLOW_SPEC_ETH | inner_bit)) &&
 599		    ib_spec->eth.mask.ether_type) {
 600			mask_valid = (ib_spec->eth.mask.ether_type ==
 601				      htons(0xffff));
 602			has_ethertype = true;
 603			eth_type = ntohs(ib_spec->eth.val.ether_type);
 604		} else if ((ib_spec->type == (IB_FLOW_SPEC_IPV4 | inner_bit)) ||
 605			   (ib_spec->type == (IB_FLOW_SPEC_IPV6 | inner_bit))) {
 606			ip_spec_type = ib_spec->type;
 607		}
 608		ib_spec = (void *)ib_spec + ib_spec->size;
 609	}
 610
 611	type_valid = (!has_ethertype) || (!ip_spec_type);
 612	if (!type_valid && mask_valid) {
 613		ipv4_spec_valid = (eth_type == ETH_P_IP) &&
 614			(ip_spec_type == (IB_FLOW_SPEC_IPV4 | inner_bit));
 615		ipv6_spec_valid = (eth_type == ETH_P_IPV6) &&
 616			(ip_spec_type == (IB_FLOW_SPEC_IPV6 | inner_bit));
 617
 618		type_valid = (ipv4_spec_valid) || (ipv6_spec_valid) ||
 619			     (((eth_type == ETH_P_MPLS_UC) ||
 620			       (eth_type == ETH_P_MPLS_MC)) && match_ipv);
 621	}
 622
 623	return type_valid;
 624}
 625
 626static bool is_valid_attr(struct mlx5_core_dev *mdev,
 627			  const struct ib_flow_attr *flow_attr)
 628{
 629	return is_valid_ethertype(mdev, flow_attr, false) &&
 630	       is_valid_ethertype(mdev, flow_attr, true);
 631}
 632
 633static void put_flow_table(struct mlx5_ib_dev *dev,
 634			   struct mlx5_ib_flow_prio *prio, bool ft_added)
 635{
 636	prio->refcount -= !!ft_added;
 637	if (!prio->refcount) {
 638		mlx5_destroy_flow_table(prio->flow_table);
 639		prio->flow_table = NULL;
 640	}
 641}
 642
 643static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
 644{
 645	struct mlx5_ib_flow_handler *handler = container_of(flow_id,
 646							  struct mlx5_ib_flow_handler,
 647							  ibflow);
 648	struct mlx5_ib_flow_handler *iter, *tmp;
 649	struct mlx5_ib_dev *dev = handler->dev;
 650
 651	mutex_lock(&dev->flow_db->lock);
 652
 653	list_for_each_entry_safe(iter, tmp, &handler->list, list) {
 654		mlx5_del_flow_rules(iter->rule);
 655		put_flow_table(dev, iter->prio, true);
 656		list_del(&iter->list);
 657		kfree(iter);
 658	}
 659
 660	mlx5_del_flow_rules(handler->rule);
 661	put_flow_table(dev, handler->prio, true);
 662	mlx5_ib_counters_clear_description(handler->ibcounters);
 663	mutex_unlock(&dev->flow_db->lock);
 664	if (handler->flow_matcher)
 665		atomic_dec(&handler->flow_matcher->usecnt);
 666	kfree(handler);
 667
 668	return 0;
 669}
 670
 671static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
 672{
 673	priority *= 2;
 674	if (!dont_trap)
 675		priority++;
 676	return priority;
 677}
 678
 679enum flow_table_type {
 680	MLX5_IB_FT_RX,
 681	MLX5_IB_FT_TX
 682};
 683
 684#define MLX5_FS_MAX_TYPES	 6
 685#define MLX5_FS_MAX_ENTRIES	 BIT(16)
 686
 687static bool __maybe_unused mlx5_ib_shared_ft_allowed(struct ib_device *device)
 688{
 689	struct mlx5_ib_dev *dev = to_mdev(device);
 690
 691	return MLX5_CAP_GEN(dev->mdev, shared_object_to_user_object_allowed);
 692}
 693
 694static struct mlx5_ib_flow_prio *_get_prio(struct mlx5_flow_namespace *ns,
 695					   struct mlx5_ib_flow_prio *prio,
 696					   struct mlx5_flow_table_attr *ft_attr)
 697{
 698	struct mlx5_flow_table *ft;
 699
 700	ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr);
 701	if (IS_ERR(ft))
 702		return ERR_CAST(ft);
 703
 704	prio->flow_table = ft;
 705	prio->refcount = 0;
 706	return prio;
 707}
 708
 709static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 710						struct ib_flow_attr *flow_attr,
 711						enum flow_table_type ft_type)
 712{
 713	bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
 714	struct mlx5_flow_table_attr ft_attr = {};
 715	struct mlx5_flow_namespace *ns = NULL;
 716	enum mlx5_flow_namespace_type fn_type;
 717	struct mlx5_ib_flow_prio *prio;
 718	struct mlx5_flow_table *ft;
 719	int max_table_size;
 720	int num_entries;
 721	int num_groups;
 722	bool esw_encap;
 723	u32 flags = 0;
 724	int priority;
 725
 726	max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
 727						       log_max_ft_size));
 728	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
 729		DEVLINK_ESWITCH_ENCAP_MODE_NONE;
 730	switch (flow_attr->type) {
 731	case IB_FLOW_ATTR_NORMAL:
 732		if (flow_is_multicast_only(flow_attr) && !dont_trap)
 733			priority = MLX5_IB_FLOW_MCAST_PRIO;
 734		else
 735			priority = ib_prio_to_core_prio(flow_attr->priority,
 736							dont_trap);
 737		if (ft_type == MLX5_IB_FT_RX) {
 738			fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
 739			prio = &dev->flow_db->prios[priority];
 740			if (!dev->is_rep && !esw_encap &&
 741			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
 742				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
 743			if (!dev->is_rep && !esw_encap &&
 744			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
 745						      reformat_l3_tunnel_to_l2))
 746				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
 747		} else {
 748			max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(
 749				dev->mdev, log_max_ft_size));
 750			fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
 751			prio = &dev->flow_db->egress_prios[priority];
 752			if (!dev->is_rep && !esw_encap &&
 753			    MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
 754				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
 755		}
 756		ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
 757		num_entries = MLX5_FS_MAX_ENTRIES;
 758		num_groups = MLX5_FS_MAX_TYPES;
 759		break;
 760	case IB_FLOW_ATTR_ALL_DEFAULT:
 761	case IB_FLOW_ATTR_MC_DEFAULT:
 762		ns = mlx5_get_flow_namespace(dev->mdev,
 763					     MLX5_FLOW_NAMESPACE_LEFTOVERS);
 764		build_leftovers_ft_param(&priority, &num_entries, &num_groups);
 765		prio = &dev->flow_db->prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
 766		break;
 767	case IB_FLOW_ATTR_SNIFFER:
 768		if (!MLX5_CAP_FLOWTABLE(dev->mdev,
 769					allow_sniffer_and_nic_rx_shared_tir))
 770			return ERR_PTR(-EOPNOTSUPP);
 771
 772		ns = mlx5_get_flow_namespace(
 773			dev->mdev, ft_type == MLX5_IB_FT_RX ?
 774					   MLX5_FLOW_NAMESPACE_SNIFFER_RX :
 775					   MLX5_FLOW_NAMESPACE_SNIFFER_TX);
 776
 777		prio = &dev->flow_db->sniffer[ft_type];
 778		priority = 0;
 779		num_entries = 1;
 780		num_groups = 1;
 781		break;
 782	default:
 783		break;
 784	}
 785
 786	if (!ns)
 787		return ERR_PTR(-EOPNOTSUPP);
 788
 789	max_table_size = min_t(int, num_entries, max_table_size);
 790
 791	ft = prio->flow_table;
 792	if (ft)
 793		return prio;
 794
 795	ft_attr.prio = priority;
 796	ft_attr.max_fte = max_table_size;
 797	ft_attr.flags = flags;
 798	ft_attr.autogroup.max_num_groups = num_groups;
 799	return _get_prio(ns, prio, &ft_attr);
 800}
 801
 802enum {
 803	RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO,
 804	RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO,
 805	RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
 806	RDMA_RX_ECN_OPCOUNTER_PRIO,
 807	RDMA_RX_CNP_OPCOUNTER_PRIO,
 808	RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO,
 809};
 810
 811enum {
 812	RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO,
 813	RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO,
 814	RDMA_TX_CNP_OPCOUNTER_PRIO,
 815	RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO,
 816};
 817
 818static int set_vhca_port_spec(struct mlx5_ib_dev *dev, u32 port_num,
 819			      struct mlx5_flow_spec *spec)
 820{
 821	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
 822					ft_field_support.source_vhca_port) ||
 823	    !MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
 824					ft_field_support.source_vhca_port))
 825		return -EOPNOTSUPP;
 826
 827	MLX5_SET_TO_ONES(fte_match_param, &spec->match_criteria,
 828			 misc_parameters.source_vhca_port);
 829	MLX5_SET(fte_match_param, &spec->match_value,
 830		 misc_parameters.source_vhca_port, port_num);
 831
 832	return 0;
 833}
 834
 835static int set_ecn_ce_spec(struct mlx5_ib_dev *dev, u32 port_num,
 836			   struct mlx5_flow_spec *spec, int ipv)
 837{
 838	if (!MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev,
 839					ft_field_support.outer_ip_version))
 840		return -EOPNOTSUPP;
 841
 842	if (mlx5_core_mp_enabled(dev->mdev) &&
 843	    set_vhca_port_spec(dev, port_num, spec))
 844		return -EOPNOTSUPP;
 845
 846	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 847			 outer_headers.ip_ecn);
 848	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_ecn,
 849		 INET_ECN_CE);
 850	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 851			 outer_headers.ip_version);
 852	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
 853		 ipv);
 854
 855	spec->match_criteria_enable =
 856		get_match_criteria_enable(spec->match_criteria);
 857
 858	return 0;
 859}
 860
 861static int set_cnp_spec(struct mlx5_ib_dev *dev, u32 port_num,
 862			struct mlx5_flow_spec *spec)
 863{
 864	if (mlx5_core_mp_enabled(dev->mdev) &&
 865	    set_vhca_port_spec(dev, port_num, spec))
 866		return -EOPNOTSUPP;
 867
 868	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
 869			 misc_parameters.bth_opcode);
 870	MLX5_SET(fte_match_param, spec->match_value, misc_parameters.bth_opcode,
 871		 IB_BTH_OPCODE_CNP);
 872
 873	spec->match_criteria_enable =
 874		get_match_criteria_enable(spec->match_criteria);
 875
 876	return 0;
 877}
 878
 879/* Returns the prio we should use for the given optional counter type,
 880 * whereas for bytes type we use the packet type, since they share the same
 881 * resources.
 882 */
 883static struct mlx5_ib_flow_prio *get_opfc_prio(struct mlx5_ib_dev *dev,
 884					       u32 type)
 885{
 886	u32 prio_type;
 887
 888	switch (type) {
 889	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
 890		prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
 891		break;
 892	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
 893		prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
 894		break;
 895	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
 896		prio_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
 897		break;
 898	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
 899		prio_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
 900		break;
 901	default:
 902		prio_type = type;
 903	}
 904
 905	return &dev->flow_db->opfcs[prio_type];
 906}
 907
 908static void put_per_qp_prio(struct mlx5_ib_dev *dev,
 909			    enum mlx5_ib_optional_counter_type type)
 910{
 911	enum mlx5_ib_optional_counter_type per_qp_type;
 912	struct mlx5_ib_flow_prio *prio;
 913
 914	switch (type) {
 915	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
 916		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
 917		break;
 918	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
 919		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
 920		break;
 921	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
 922		per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
 923		break;
 924	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
 925		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
 926		break;
 927	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
 928		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
 929		break;
 930	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
 931		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
 932		break;
 933	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
 934		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
 935		break;
 936	default:
 937		return;
 938	}
 939
 940	prio = get_opfc_prio(dev, per_qp_type);
 941	put_flow_table(dev, prio, true);
 942}
 943
 944static int get_per_qp_prio(struct mlx5_ib_dev *dev,
 945			   enum mlx5_ib_optional_counter_type type)
 946{
 947	enum mlx5_ib_optional_counter_type per_qp_type;
 948	struct mlx5_flow_table_attr ft_attr = {};
 949	enum mlx5_flow_namespace_type fn_type;
 950	struct mlx5_flow_namespace *ns;
 951	struct mlx5_ib_flow_prio *prio;
 952	int priority;
 953
 954	switch (type) {
 955	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
 956		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
 957		priority = RDMA_RX_ECN_OPCOUNTER_PER_QP_PRIO;
 958		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
 959		break;
 960	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
 961		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
 962		priority = RDMA_RX_CNP_OPCOUNTER_PER_QP_PRIO;
 963		per_qp_type = MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP;
 964		break;
 965	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
 966		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
 967		priority = RDMA_TX_CNP_OPCOUNTER_PER_QP_PRIO;
 968		per_qp_type = MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP;
 969		break;
 970	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
 971		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
 972		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
 973		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
 974		break;
 975	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
 976		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
 977		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
 978		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
 979		break;
 980	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
 981		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
 982		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
 983		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
 984		break;
 985	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
 986		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
 987		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PER_QP_PRIO;
 988		per_qp_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
 989		break;
 990	default:
 991		return -EINVAL;
 992	}
 993
 994	ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
 995	if (!ns)
 996		return -EOPNOTSUPP;
 997
 998	prio = get_opfc_prio(dev, per_qp_type);
 999	if (prio->flow_table)
1000		return 0;
1001
1002	ft_attr.prio = priority;
1003	ft_attr.max_fte = MLX5_FS_MAX_POOL_SIZE;
1004	ft_attr.autogroup.max_num_groups = 1;
1005	prio = _get_prio(ns, prio, &ft_attr);
1006	if (IS_ERR(prio))
1007		return PTR_ERR(prio);
1008
1009	prio->refcount = 1;
1010
1011	return 0;
1012}
1013
1014static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,
1015						u32 qp_num, bool *new)
1016{
1017	struct mlx5_per_qp_opfc *per_qp_opfc;
1018
1019	*new = false;
1020
1021	per_qp_opfc = xa_load(qpn_opfc_xa, qp_num);
1022	if (per_qp_opfc)
1023		return per_qp_opfc;
1024	per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
1025
1026	if (!per_qp_opfc)
1027		return NULL;
1028
1029	*new = true;
1030	return per_qp_opfc;
1031}
1032
1033static int add_op_fc_rules(struct mlx5_ib_dev *dev,
1034			   struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1035			   struct xarray *qpn_opfc_xa,
1036			   struct mlx5_per_qp_opfc *per_qp_opfc,
1037			   struct mlx5_ib_flow_prio *prio,
1038			   enum mlx5_ib_optional_counter_type type,
1039			   u32 qp_num, u32 port_num)
1040{
1041	struct mlx5_ib_op_fc *opfc = &per_qp_opfc->opfcs[type], *in_use_opfc;
1042	struct mlx5_flow_act flow_act = {};
1043	struct mlx5_flow_destination dst;
1044	struct mlx5_flow_spec *spec;
1045	int i, err, spec_num;
1046	bool is_tx;
1047
1048	if (opfc->fc)
1049		return -EEXIST;
1050
1051	if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, type,
1052					    &in_use_opfc)) {
1053		opfc->fc = in_use_opfc->fc;
1054		opfc->rule[0] = in_use_opfc->rule[0];
1055		return 0;
1056	}
1057
1058	opfc->fc = fc_arr[type];
1059
1060	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1061	if (!spec) {
1062		err = -ENOMEM;
1063		goto null_fc;
1064	}
1065
1066	switch (type) {
1067	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP:
1068		if (set_ecn_ce_spec(dev, port_num, &spec[0],
1069				    MLX5_FS_IPV4_VERSION) ||
1070		    set_ecn_ce_spec(dev, port_num, &spec[1],
1071				    MLX5_FS_IPV6_VERSION)) {
1072			err = -EOPNOTSUPP;
1073			goto free_spec;
1074		}
1075		spec_num = 2;
1076		is_tx = false;
1077
1078		MLX5_SET_TO_ONES(fte_match_param, spec[1].match_criteria,
1079				 misc_parameters.bth_dst_qp);
1080		MLX5_SET(fte_match_param, spec[1].match_value,
1081			 misc_parameters.bth_dst_qp, qp_num);
1082		spec[1].match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1083		break;
1084	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS_PER_QP:
1085		if (!MLX5_CAP_FLOWTABLE(
1086			    dev->mdev,
1087			    ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1088		    set_cnp_spec(dev, port_num, &spec[0])) {
1089			err = -EOPNOTSUPP;
1090			goto free_spec;
1091		}
1092		spec_num = 1;
1093		is_tx = false;
1094		break;
1095	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS_PER_QP:
1096		if (!MLX5_CAP_FLOWTABLE(
1097			    dev->mdev,
1098			    ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1099		    set_cnp_spec(dev, port_num, &spec[0])) {
1100			err = -EOPNOTSUPP;
1101			goto free_spec;
1102		}
1103		spec_num = 1;
1104		is_tx = true;
1105		break;
1106	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1107	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1108		spec_num = 1;
1109		is_tx = true;
1110		break;
1111	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1112	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1113		spec_num = 1;
1114		is_tx = false;
1115		break;
1116	default:
1117		err = -EINVAL;
1118		goto free_spec;
1119	}
1120
1121	if (is_tx) {
1122		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1123				 misc_parameters.source_sqn);
1124		MLX5_SET(fte_match_param, spec->match_value,
1125			 misc_parameters.source_sqn, qp_num);
1126	} else {
1127		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
1128				 misc_parameters.bth_dst_qp);
1129		MLX5_SET(fte_match_param, spec->match_value,
1130			 misc_parameters.bth_dst_qp, qp_num);
1131	}
1132
1133	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
1134
1135	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1136	dst.counter = opfc->fc;
1137
1138	flow_act.action =
1139		MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1140
1141	for (i = 0; i < spec_num; i++) {
1142		opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1143						    &flow_act, &dst, 1);
1144		if (IS_ERR(opfc->rule[i])) {
1145			err = PTR_ERR(opfc->rule[i]);
1146			goto del_rules;
1147		}
1148	}
1149	prio->refcount += spec_num;
1150
1151	err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL));
1152	if (err)
1153		goto del_rules;
1154
1155	kfree(spec);
1156
1157	return 0;
1158
1159del_rules:
1160	while (i--)
1161		mlx5_del_flow_rules(opfc->rule[i]);
1162	put_flow_table(dev, prio, false);
1163free_spec:
1164	kfree(spec);
1165null_fc:
1166	opfc->fc = NULL;
1167	return err;
1168}
1169
1170static bool
1171is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,
1172			struct mlx5_fc **fc)
1173{
1174	u32 shared_fc_type;
1175
1176	switch (type) {
1177	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
1178		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
1179		break;
1180	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
1181		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
1182		break;
1183	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
1184		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
1185		break;
1186	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
1187		shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
1188		break;
1189	default:
1190		return false;
1191	}
1192
1193	*fc = fc_arr[shared_fc_type];
1194	if (!(*fc))
1195		return false;
1196
1197	return true;
1198}
1199
1200void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
1201			  struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])
1202{
1203	struct mlx5_fc *in_use_fc;
1204	int i;
1205
1206	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1207	     i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1208		if (!fc_arr[i])
1209			continue;
1210
1211		if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) {
1212			fc_arr[i] = NULL;
1213			continue;
1214		}
1215
1216		mlx5_fc_destroy(dev->mdev, fc_arr[i]);
1217		fc_arr[i] = NULL;
1218	}
1219}
1220
1221int mlx5_ib_fs_add_op_fc(struct mlx5_ib_dev *dev, u32 port_num,
1222			 struct mlx5_ib_op_fc *opfc,
1223			 enum mlx5_ib_optional_counter_type type)
1224{
1225	struct mlx5_flow_table_attr ft_attr = {};
1226	enum mlx5_flow_namespace_type fn_type;
1227	int priority, i, err, spec_num;
1228	struct mlx5_flow_act flow_act = {};
1229	struct mlx5_flow_destination dst;
1230	struct mlx5_flow_namespace *ns;
1231	struct mlx5_ib_flow_prio *prio;
1232	struct mlx5_flow_spec *spec;
1233
1234	spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
1235	if (!spec)
1236		return -ENOMEM;
1237
1238	switch (type) {
1239	case MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS:
1240		if (set_ecn_ce_spec(dev, port_num, &spec[0],
1241				    MLX5_FS_IPV4_VERSION) ||
1242		    set_ecn_ce_spec(dev, port_num, &spec[1],
1243				    MLX5_FS_IPV6_VERSION)) {
1244			err = -EOPNOTSUPP;
1245			goto free;
1246		}
1247		spec_num = 2;
1248		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1249		priority = RDMA_RX_ECN_OPCOUNTER_PRIO;
1250		break;
1251
1252	case MLX5_IB_OPCOUNTER_CC_RX_CNP_PKTS:
1253		if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1254					ft_field_support_2_nic_receive_rdma.bth_opcode) ||
1255		    set_cnp_spec(dev, port_num, &spec[0])) {
1256			err = -EOPNOTSUPP;
1257			goto free;
1258		}
1259		spec_num = 1;
1260		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1261		priority = RDMA_RX_CNP_OPCOUNTER_PRIO;
1262		break;
1263
1264	case MLX5_IB_OPCOUNTER_CC_TX_CNP_PKTS:
1265		if (!MLX5_CAP_FLOWTABLE(dev->mdev,
1266					ft_field_support_2_nic_transmit_rdma.bth_opcode) ||
1267		    set_cnp_spec(dev, port_num, &spec[0])) {
1268			err = -EOPNOTSUPP;
1269			goto free;
1270		}
1271		spec_num = 1;
1272		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1273		priority = RDMA_TX_CNP_OPCOUNTER_PRIO;
1274		break;
1275
1276	case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
1277	case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
1278		spec_num = 1;
1279		fn_type = MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS;
1280		priority = RDMA_TX_PKTS_BYTES_OPCOUNTER_PRIO;
1281		break;
1282
1283	case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
1284	case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
1285		spec_num = 1;
1286		fn_type = MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS;
1287		priority = RDMA_RX_PKTS_BYTES_OPCOUNTER_PRIO;
1288		break;
1289
1290	default:
1291		err = -EOPNOTSUPP;
1292		goto free;
1293	}
1294
1295	ns = mlx5_get_flow_namespace(dev->mdev, fn_type);
1296	if (!ns) {
1297		err = -EOPNOTSUPP;
1298		goto free;
1299	}
1300
1301	prio = get_opfc_prio(dev, type);
1302	if (!prio->flow_table) {
1303		err = get_per_qp_prio(dev, type);
1304		if (err)
1305			goto free;
1306
1307		ft_attr.prio = priority;
1308		ft_attr.max_fte = dev->num_ports * MAX_OPFC_RULES;
1309		ft_attr.autogroup.max_num_groups = 1;
1310		prio = _get_prio(ns, prio, &ft_attr);
1311		if (IS_ERR(prio)) {
1312			err = PTR_ERR(prio);
1313			goto put_prio;
1314		}
1315	}
1316
1317	dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1318	dst.counter = opfc->fc;
1319
1320	flow_act.action =
1321		MLX5_FLOW_CONTEXT_ACTION_COUNT | MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1322
1323	for (i = 0; i < spec_num; i++) {
1324		opfc->rule[i] = mlx5_add_flow_rules(prio->flow_table, &spec[i],
1325						    &flow_act, &dst, 1);
1326		if (IS_ERR(opfc->rule[i])) {
1327			err = PTR_ERR(opfc->rule[i]);
1328			goto del_rules;
1329		}
1330	}
1331	prio->refcount += spec_num;
1332	kfree(spec);
1333
1334	return 0;
1335
1336del_rules:
1337	for (i -= 1; i >= 0; i--)
1338		mlx5_del_flow_rules(opfc->rule[i]);
1339	put_flow_table(dev, prio, false);
1340put_prio:
1341	put_per_qp_prio(dev, type);
1342free:
1343	kfree(spec);
1344	return err;
1345}
1346
1347void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
1348			     struct mlx5_ib_op_fc *opfc,
1349			     enum mlx5_ib_optional_counter_type type)
1350{
1351	struct mlx5_ib_flow_prio *prio;
1352	int i;
1353
1354	prio = get_opfc_prio(dev, type);
1355
1356	for (i = 0; i < MAX_OPFC_RULES && opfc->rule[i]; i++) {
1357		mlx5_del_flow_rules(opfc->rule[i]);
1358		put_flow_table(dev, prio, true);
1359	}
1360
1361	put_per_qp_prio(dev, type);
1362}
1363
1364void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)
1365{
1366	struct mlx5_ib_dev *dev = to_mdev(qp->device);
1367	struct mlx5_per_qp_opfc *per_qp_opfc;
1368	struct mlx5_ib_op_fc *in_use_opfc;
1369	struct mlx5_ib_flow_prio *prio;
1370	int i, j;
1371
1372	per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num);
1373	if (!per_qp_opfc)
1374		return;
1375
1376	for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1377	     i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
1378		if (!per_qp_opfc->opfcs[i].fc)
1379			continue;
1380
1381		if (mlx5r_is_opfc_shared_and_in_use(per_qp_opfc->opfcs, i,
1382						    &in_use_opfc)) {
1383			per_qp_opfc->opfcs[i].fc = NULL;
1384			continue;
1385		}
1386
1387		for (j = 0; j < MAX_OPFC_RULES; j++) {
1388			if (!per_qp_opfc->opfcs[i].rule[j])
1389				continue;
1390			mlx5_del_flow_rules(per_qp_opfc->opfcs[i].rule[j]);
1391			prio = get_opfc_prio(dev, i);
1392			put_flow_table(dev, prio, true);
1393		}
1394		per_qp_opfc->opfcs[i].fc = NULL;
1395	}
1396
1397	kfree(per_qp_opfc);
1398	xa_erase(qpn_opfc_xa, qp->qp_num);
1399}
1400
1401int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
1402			struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
1403			struct xarray *qpn_opfc_xa, u32 port)
1404{
1405	struct mlx5_ib_dev *dev = to_mdev(qp->device);
1406	struct mlx5_per_qp_opfc *per_qp_opfc;
1407	struct mlx5_ib_flow_prio *prio;
1408	struct mlx5_ib_counters *cnts;
1409	struct mlx5_ib_op_fc *opfc;
1410	struct mlx5_fc *in_use_fc;
1411	int i, err, per_qp_type;
1412	bool new;
1413
1414	cnts = &dev->port[port - 1].cnts;
1415
1416	for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
1417		opfc = &cnts->opfcs[i];
1418		if (!opfc->fc)
1419			continue;
1420
1421		per_qp_type = i + MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
1422		prio = get_opfc_prio(dev, per_qp_type);
1423		WARN_ON(!prio->flow_table);
1424
1425		if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc))
1426			fc_arr[per_qp_type] = in_use_fc;
1427
1428		if (!fc_arr[per_qp_type]) {
1429			fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false);
1430			if (IS_ERR(fc_arr[per_qp_type]))
1431				return PTR_ERR(fc_arr[per_qp_type]);
1432		}
1433
1434		per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new);
1435		if (!per_qp_opfc) {
1436			err = -ENOMEM;
1437			goto free_fc;
1438		}
1439		err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,
1440				      prio, per_qp_type, qp->qp_num, port);
1441		if (err)
1442			goto del_rules;
1443	}
1444
1445	return 0;
1446
1447del_rules:
1448	mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);
1449	if (new)
1450		kfree(per_qp_opfc);
1451free_fc:
1452	if (xa_empty(qpn_opfc_xa))
1453		mlx5r_fs_destroy_fcs(dev, fc_arr);
1454	return err;
1455}
1456
1457static void set_underlay_qp(struct mlx5_ib_dev *dev,
1458			    struct mlx5_flow_spec *spec,
1459			    u32 underlay_qpn)
1460{
1461	void *misc_params_c = MLX5_ADDR_OF(fte_match_param,
1462					   spec->match_criteria,
1463					   misc_parameters);
1464	void *misc_params_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1465					   misc_parameters);
1466
1467	if (underlay_qpn &&
1468	    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1469				      ft_field_support.bth_dst_qp)) {
1470		MLX5_SET(fte_match_set_misc,
1471			 misc_params_v, bth_dst_qp, underlay_qpn);
1472		MLX5_SET(fte_match_set_misc,
1473			 misc_params_c, bth_dst_qp, 0xffffff);
1474	}
1475}
1476
1477static void mlx5_ib_set_rule_source_port(struct mlx5_ib_dev *dev,
1478					 struct mlx5_flow_spec *spec,
1479					 struct mlx5_eswitch_rep *rep)
1480{
1481	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
1482	void *misc;
1483
1484	if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1485		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1486				    misc_parameters_2);
1487
1488		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1489			 mlx5_eswitch_get_vport_metadata_for_match(rep->esw,
1490								   rep->vport));
1491		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1492				    misc_parameters_2);
1493
1494		MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1495			 mlx5_eswitch_get_vport_metadata_mask());
1496	} else {
1497		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1498				    misc_parameters);
1499
1500		MLX5_SET(fte_match_set_misc, misc, source_port, rep->vport);
1501
1502		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1503				    misc_parameters);
1504
1505		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1506	}
1507}
1508
1509static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
1510						      struct mlx5_ib_flow_prio *ft_prio,
1511						      const struct ib_flow_attr *flow_attr,
1512						      struct mlx5_flow_destination *dst,
1513						      u32 underlay_qpn,
1514						      struct mlx5_ib_create_flow *ucmd)
1515{
1516	struct mlx5_flow_table	*ft = ft_prio->flow_table;
1517	struct mlx5_ib_flow_handler *handler;
1518	struct mlx5_flow_act flow_act = {};
1519	struct mlx5_flow_spec *spec;
1520	struct mlx5_flow_destination dest_arr[2] = {};
1521	struct mlx5_flow_destination *rule_dst = dest_arr;
1522	const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
1523	unsigned int spec_index;
1524	u32 prev_type = 0;
1525	int err = 0;
1526	int dest_num = 0;
1527	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1528
1529	if (!is_valid_attr(dev->mdev, flow_attr))
1530		return ERR_PTR(-EINVAL);
1531
1532	if (dev->is_rep && is_egress)
1533		return ERR_PTR(-EINVAL);
1534
1535	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1536	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
1537	if (!handler || !spec) {
1538		err = -ENOMEM;
1539		goto free;
1540	}
1541
1542	INIT_LIST_HEAD(&handler->list);
1543
1544	for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
1545		err = parse_flow_attr(dev->mdev, spec,
1546				      ib_flow, flow_attr, &flow_act,
1547				      prev_type);
1548		if (err < 0)
1549			goto free;
1550
1551		prev_type = ((union ib_flow_spec *)ib_flow)->type;
1552		ib_flow += ((union ib_flow_spec *)ib_flow)->size;
1553	}
1554
1555	if (dst && !(flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP)) {
1556		memcpy(&dest_arr[0], dst, sizeof(*dst));
1557		dest_num++;
1558	}
1559
1560	if (!flow_is_multicast_only(flow_attr))
1561		set_underlay_qp(dev, spec, underlay_qpn);
1562
1563	if (dev->is_rep && flow_attr->type != IB_FLOW_ATTR_SNIFFER) {
1564		struct mlx5_eswitch_rep *rep;
1565
1566		rep = dev->port[flow_attr->port - 1].rep;
1567		if (!rep) {
1568			err = -EINVAL;
1569			goto free;
1570		}
1571
1572		mlx5_ib_set_rule_source_port(dev, spec, rep);
1573	}
1574
1575	spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
1576
1577	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1578		struct mlx5_ib_mcounters *mcounters;
1579
1580		err = mlx5_ib_flow_counters_set_data(flow_act.counters, ucmd);
1581		if (err)
1582			goto free;
1583
1584		mcounters = to_mcounters(flow_act.counters);
1585		handler->ibcounters = flow_act.counters;
1586		dest_arr[dest_num].type =
1587			MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1588		dest_arr[dest_num].counter =
1589			mcounters->hw_cntrs_hndl;
1590		dest_num++;
1591	}
1592
1593	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1594		if (!dest_num)
1595			rule_dst = NULL;
1596	} else {
1597		if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP)
1598			flow_act.action |=
1599				MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
1600		if (is_egress)
1601			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
1602		else if (dest_num)
1603			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1604	}
1605
1606	if ((spec->flow_context.flags & FLOW_CONTEXT_HAS_TAG)  &&
1607	    (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1608	     flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1609		mlx5_ib_warn(dev, "Flow tag %u and attribute type %x isn't allowed in leftovers\n",
1610			     spec->flow_context.flow_tag, flow_attr->type);
1611		err = -EINVAL;
1612		goto free;
1613	}
1614	handler->rule = mlx5_add_flow_rules(ft, spec,
1615					    &flow_act,
1616					    rule_dst, dest_num);
1617
1618	if (IS_ERR(handler->rule)) {
1619		err = PTR_ERR(handler->rule);
1620		goto free;
1621	}
1622
1623	ft_prio->refcount++;
1624	handler->prio = ft_prio;
1625	handler->dev = dev;
1626
1627	ft_prio->flow_table = ft;
1628free:
1629	if (err && handler) {
1630		mlx5_ib_counters_clear_description(handler->ibcounters);
1631		kfree(handler);
1632	}
1633	kvfree(spec);
1634	return err ? ERR_PTR(err) : handler;
1635}
1636
1637static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
1638						     struct mlx5_ib_flow_prio *ft_prio,
1639						     const struct ib_flow_attr *flow_attr,
1640						     struct mlx5_flow_destination *dst)
1641{
1642	return _create_flow_rule(dev, ft_prio, flow_attr, dst, 0, NULL);
1643}
1644
1645static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
1646							  struct mlx5_ib_flow_prio *ft_prio,
1647							  struct ib_flow_attr *flow_attr,
1648							  struct mlx5_flow_destination *dst)
1649{
1650	struct mlx5_ib_flow_handler *handler_ucast = NULL;
1651	struct mlx5_ib_flow_handler *handler = NULL;
1652
1653	static struct {
1654		struct ib_flow_spec_eth eth_flow;
1655		struct ib_flow_attr	flow_attr;
1656	} leftovers_wc = { .flow_attr = { .num_of_specs = 1,
1657					  .size = sizeof(leftovers_wc) },
1658			   .eth_flow = {
1659				   .type = IB_FLOW_SPEC_ETH,
1660				   .size = sizeof(struct ib_flow_spec_eth),
1661				   .mask = { .dst_mac = { 0x1 } },
1662				   .val = { .dst_mac = { 0x1 } } } };
1663
1664	static struct {
1665		struct ib_flow_spec_eth eth_flow;
1666		struct ib_flow_attr	flow_attr;
1667	} leftovers_uc = { .flow_attr = { .num_of_specs = 1,
1668					  .size = sizeof(leftovers_uc) },
1669			   .eth_flow = {
1670				   .type = IB_FLOW_SPEC_ETH,
1671				   .size = sizeof(struct ib_flow_spec_eth),
1672				   .mask = { .dst_mac = { 0x1 } },
1673				   .val = { .dst_mac = {} } } };
1674
1675	handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
1676	if (!IS_ERR(handler) &&
1677	    flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
1678		handler_ucast = create_flow_rule(dev, ft_prio,
1679						 &leftovers_uc.flow_attr, dst);
1680		if (IS_ERR(handler_ucast)) {
1681			mlx5_del_flow_rules(handler->rule);
1682			ft_prio->refcount--;
1683			kfree(handler);
1684			handler = handler_ucast;
1685		} else {
1686			list_add(&handler_ucast->list, &handler->list);
1687		}
1688	}
1689
1690	return handler;
1691}
1692
1693static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
1694							struct mlx5_ib_flow_prio *ft_rx,
1695							struct mlx5_ib_flow_prio *ft_tx,
1696							struct mlx5_flow_destination *dst)
1697{
1698	struct mlx5_ib_flow_handler *handler_rx;
1699	struct mlx5_ib_flow_handler *handler_tx;
1700	int err;
1701	static const struct ib_flow_attr flow_attr  = {
1702		.num_of_specs = 0,
1703		.type = IB_FLOW_ATTR_SNIFFER,
1704		.size = sizeof(flow_attr)
1705	};
1706
1707	handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
1708	if (IS_ERR(handler_rx)) {
1709		err = PTR_ERR(handler_rx);
1710		goto err;
1711	}
1712
1713	handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
1714	if (IS_ERR(handler_tx)) {
1715		err = PTR_ERR(handler_tx);
1716		goto err_tx;
1717	}
1718
1719	list_add(&handler_tx->list, &handler_rx->list);
1720
1721	return handler_rx;
1722
1723err_tx:
1724	mlx5_del_flow_rules(handler_rx->rule);
1725	ft_rx->refcount--;
1726	kfree(handler_rx);
1727err:
1728	return ERR_PTR(err);
1729}
1730
1731static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
1732					   struct ib_flow_attr *flow_attr,
1733					   struct ib_udata *udata)
1734{
1735	struct mlx5_ib_dev *dev = to_mdev(qp->device);
1736	struct mlx5_ib_qp *mqp = to_mqp(qp);
1737	struct mlx5_ib_flow_handler *handler = NULL;
1738	struct mlx5_flow_destination *dst = NULL;
1739	struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
1740	struct mlx5_ib_flow_prio *ft_prio;
1741	bool is_egress = flow_attr->flags & IB_FLOW_ATTR_FLAGS_EGRESS;
1742	struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
1743	size_t min_ucmd_sz, required_ucmd_sz;
1744	int err;
1745	int underlay_qpn;
1746
1747	if (udata && udata->inlen) {
1748		min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
1749		if (udata->inlen < min_ucmd_sz)
1750			return ERR_PTR(-EOPNOTSUPP);
1751
1752		err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
1753		if (err)
1754			return ERR_PTR(err);
1755
1756		/* currently supports only one counters data */
1757		if (ucmd_hdr.ncounters_data > 1)
1758			return ERR_PTR(-EINVAL);
1759
1760		required_ucmd_sz = min_ucmd_sz +
1761			sizeof(struct mlx5_ib_flow_counters_data) *
1762			ucmd_hdr.ncounters_data;
1763		if (udata->inlen > required_ucmd_sz &&
1764		    !ib_is_udata_cleared(udata, required_ucmd_sz,
1765					 udata->inlen - required_ucmd_sz))
1766			return ERR_PTR(-EOPNOTSUPP);
1767
1768		ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
1769		if (!ucmd)
1770			return ERR_PTR(-ENOMEM);
1771
1772		err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
1773		if (err)
1774			goto free_ucmd;
1775	}
1776
1777	if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
1778		err = -ENOMEM;
1779		goto free_ucmd;
1780	}
1781
1782	if (flow_attr->flags &
1783	    ~(IB_FLOW_ATTR_FLAGS_DONT_TRAP | IB_FLOW_ATTR_FLAGS_EGRESS)) {
1784		err = -EINVAL;
1785		goto free_ucmd;
1786	}
1787
1788	if (is_egress &&
1789	    (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
1790	     flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT)) {
1791		err = -EINVAL;
1792		goto free_ucmd;
1793	}
1794
1795	dst = kzalloc(sizeof(*dst), GFP_KERNEL);
1796	if (!dst) {
1797		err = -ENOMEM;
1798		goto free_ucmd;
1799	}
1800
1801	mutex_lock(&dev->flow_db->lock);
1802
1803	ft_prio = get_flow_table(dev, flow_attr,
1804				 is_egress ? MLX5_IB_FT_TX : MLX5_IB_FT_RX);
1805	if (IS_ERR(ft_prio)) {
1806		err = PTR_ERR(ft_prio);
1807		goto unlock;
1808	}
1809	if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
1810		ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
1811		if (IS_ERR(ft_prio_tx)) {
1812			err = PTR_ERR(ft_prio_tx);
1813			ft_prio_tx = NULL;
1814			goto destroy_ft;
1815		}
1816	}
1817
1818	if (is_egress) {
1819		dst->type = MLX5_FLOW_DESTINATION_TYPE_PORT;
1820	} else {
1821		dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1822		if (mqp->is_rss)
1823			dst->tir_num = mqp->rss_qp.tirn;
1824		else
1825			dst->tir_num = mqp->raw_packet_qp.rq.tirn;
1826	}
1827
1828	switch (flow_attr->type) {
1829	case IB_FLOW_ATTR_NORMAL:
1830		underlay_qpn = (mqp->flags & IB_QP_CREATE_SOURCE_QPN) ?
1831				       mqp->underlay_qpn :
1832				       0;
1833		handler = _create_flow_rule(dev, ft_prio, flow_attr, dst,
1834					    underlay_qpn, ucmd);
1835		break;
1836	case IB_FLOW_ATTR_ALL_DEFAULT:
1837	case IB_FLOW_ATTR_MC_DEFAULT:
1838		handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
1839		break;
1840	case IB_FLOW_ATTR_SNIFFER:
1841		handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
1842		break;
1843	default:
1844		err = -EINVAL;
1845		goto destroy_ft;
1846	}
1847
1848	if (IS_ERR(handler)) {
1849		err = PTR_ERR(handler);
1850		handler = NULL;
1851		goto destroy_ft;
1852	}
1853
1854	mutex_unlock(&dev->flow_db->lock);
1855	kfree(dst);
1856	kfree(ucmd);
1857
1858	return &handler->ibflow;
1859
1860destroy_ft:
1861	put_flow_table(dev, ft_prio, false);
1862	if (ft_prio_tx)
1863		put_flow_table(dev, ft_prio_tx, false);
1864unlock:
1865	mutex_unlock(&dev->flow_db->lock);
1866	kfree(dst);
1867free_ucmd:
1868	kfree(ucmd);
1869	return ERR_PTR(err);
1870}
1871
1872static int mlx5_ib_fill_transport_ns_info(struct mlx5_ib_dev *dev,
1873					  enum mlx5_flow_namespace_type type,
1874					  u32 *flags, u16 *vport_idx,
1875					  u16 *vport,
1876					  struct mlx5_core_dev **ft_mdev,
1877					  u32 ib_port, u16 *esw_owner_vhca_id)
1878{
1879	struct mlx5_core_dev *esw_mdev;
1880
1881	if (!is_mdev_switchdev_mode(dev->mdev))
1882		return 0;
1883
1884	if (!MLX5_CAP_ADV_RDMA(dev->mdev, rdma_transport_manager))
1885		return -EOPNOTSUPP;
1886
1887	if (!dev->port[ib_port - 1].rep)
1888		return -EINVAL;
1889
1890	esw_mdev = mlx5_eswitch_get_core_dev(dev->port[ib_port - 1].rep->esw);
1891	if (esw_mdev != dev->mdev) {
1892		if (!MLX5_CAP_ADV_RDMA(dev->mdev,
1893				       rdma_transport_manager_other_eswitch))
1894			return -EOPNOTSUPP;
1895		*flags |= MLX5_FLOW_TABLE_OTHER_ESWITCH;
1896		*esw_owner_vhca_id = MLX5_CAP_GEN(esw_mdev, vhca_id);
1897	}
1898
1899	*flags |= MLX5_FLOW_TABLE_OTHER_VPORT;
1900	*ft_mdev = esw_mdev;
1901	*vport = dev->port[ib_port - 1].rep->vport;
1902	*vport_idx = dev->port[ib_port - 1].rep->vport_index;
1903
1904	return 0;
1905}
1906
1907static struct mlx5_ib_flow_prio *
1908_get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
1909		enum mlx5_flow_namespace_type ns_type,
1910		bool mcast, u32 ib_port)
1911{
1912	struct mlx5_core_dev *ft_mdev = dev->mdev;
1913	struct mlx5_flow_table_attr ft_attr = {};
1914	struct mlx5_flow_namespace *ns = NULL;
1915	struct mlx5_ib_flow_prio *prio = NULL;
1916	u16 esw_owner_vhca_id = 0;
1917	int max_table_size = 0;
1918	u16 vport_idx = 0;
1919	bool esw_encap;
1920	u32 flags = 0;
1921	u16 vport = 0;
1922	int priority;
1923	int ret;
1924
1925	if (mcast)
1926		priority = MLX5_IB_FLOW_MCAST_PRIO;
1927	else
1928		priority = ib_prio_to_core_prio(user_priority, false);
1929
1930	esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) !=
1931		DEVLINK_ESWITCH_ENCAP_MODE_NONE;
1932	switch (ns_type) {
1933	case MLX5_FLOW_NAMESPACE_BYPASS:
1934		max_table_size = BIT(
1935			MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size));
1936		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap)
1937			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1938		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
1939					      reformat_l3_tunnel_to_l2) &&
1940		    !esw_encap)
1941			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1942		break;
1943	case MLX5_FLOW_NAMESPACE_EGRESS:
1944		max_table_size = BIT(
1945			MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
1946		if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) &&
1947		    !esw_encap)
1948			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1949		break;
1950	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
1951		max_table_size = BIT(
1952			MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
1953		if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap)
1954			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
1955		if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev,
1956					       reformat_l3_tunnel_to_l2) &&
1957		    esw_encap)
1958			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
1959		priority = user_priority;
1960		break;
1961	case MLX5_FLOW_NAMESPACE_RDMA_RX:
1962		max_table_size = BIT(
1963			MLX5_CAP_FLOWTABLE_RDMA_RX(dev->mdev, log_max_ft_size));
1964		priority = user_priority;
1965		break;
1966	case MLX5_FLOW_NAMESPACE_RDMA_TX:
1967		max_table_size = BIT(
1968			MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev, log_max_ft_size));
1969		priority = user_priority;
1970		break;
1971	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
1972	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
1973		if (ib_port == 0 ||
1974		    user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
1975			return ERR_PTR(-EINVAL);
1976		ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags,
1977						     &vport_idx, &vport,
1978						     &ft_mdev, ib_port,
1979						     &esw_owner_vhca_id);
1980		if (ret)
1981			return ERR_PTR(ret);
1982
1983		if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX)
1984			max_table_size =
1985				BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_RX(
1986					ft_mdev, log_max_ft_size));
1987		else
1988			max_table_size =
1989				BIT(MLX5_CAP_FLOWTABLE_RDMA_TRANSPORT_TX(
1990					ft_mdev, log_max_ft_size));
1991		priority = user_priority;
1992		break;
1993	default:
1994		break;
1995	}
1996
1997	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
1998
1999	if (ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX ||
2000	    ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX)
2001		ns = mlx5_get_flow_vport_namespace(ft_mdev, ns_type, vport_idx);
2002	else
2003		ns = mlx5_get_flow_namespace(ft_mdev, ns_type);
2004
2005	if (!ns)
2006		return ERR_PTR(-EOPNOTSUPP);
2007
2008	switch (ns_type) {
2009	case MLX5_FLOW_NAMESPACE_BYPASS:
2010		prio = &dev->flow_db->prios[priority];
2011		break;
2012	case MLX5_FLOW_NAMESPACE_EGRESS:
2013		prio = &dev->flow_db->egress_prios[priority];
2014		break;
2015	case MLX5_FLOW_NAMESPACE_FDB_BYPASS:
2016		prio = &dev->flow_db->fdb[priority];
2017		break;
2018	case MLX5_FLOW_NAMESPACE_RDMA_RX:
2019		prio = &dev->flow_db->rdma_rx[priority];
2020		break;
2021	case MLX5_FLOW_NAMESPACE_RDMA_TX:
2022		prio = &dev->flow_db->rdma_tx[priority];
2023		break;
2024	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
2025		prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1];
2026		break;
2027	case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
2028		prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1];
2029		break;
2030	default: return ERR_PTR(-EINVAL);
2031	}
2032
2033	if (!prio)
2034		return ERR_PTR(-EINVAL);
2035
2036	if (prio->flow_table)
2037		return prio;
2038
2039	ft_attr.prio = priority;
2040	ft_attr.max_fte = max_table_size;
2041	ft_attr.flags = flags;
2042	ft_attr.vport = vport;
2043	ft_attr.esw_owner_vhca_id = esw_owner_vhca_id;
2044	ft_attr.autogroup.max_num_groups = MLX5_FS_MAX_TYPES;
2045	return _get_prio(ns, prio, &ft_attr);
2046}
2047
2048static struct mlx5_ib_flow_handler *
2049_create_raw_flow_rule(struct mlx5_ib_dev *dev,
2050		      struct mlx5_ib_flow_prio *ft_prio,
2051		      struct mlx5_flow_destination *dst,
2052		      struct mlx5_ib_flow_matcher  *fs_matcher,
2053		      struct mlx5_flow_context *flow_context,
2054		      struct mlx5_flow_act *flow_act,
2055		      void *cmd_in, int inlen,
2056		      int dst_num)
2057{
2058	struct mlx5_ib_flow_handler *handler;
2059	struct mlx5_flow_spec *spec;
2060	struct mlx5_flow_table *ft = ft_prio->flow_table;
2061	int err = 0;
2062
2063	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2064	handler = kzalloc(sizeof(*handler), GFP_KERNEL);
2065	if (!handler || !spec) {
2066		err = -ENOMEM;
2067		goto free;
2068	}
2069
2070	INIT_LIST_HEAD(&handler->list);
2071
2072	memcpy(spec->match_value, cmd_in, inlen);
2073	memcpy(spec->match_criteria, fs_matcher->matcher_mask.match_params,
2074	       fs_matcher->mask_len);
2075	spec->match_criteria_enable = fs_matcher->match_criteria_enable;
2076	spec->flow_context = *flow_context;
2077
2078	handler->rule = mlx5_add_flow_rules(ft, spec,
2079					    flow_act, dst, dst_num);
2080
2081	if (IS_ERR(handler->rule)) {
2082		err = PTR_ERR(handler->rule);
2083		goto free;
2084	}
2085
2086	ft_prio->refcount++;
2087	handler->prio = ft_prio;
2088	handler->dev = dev;
2089	ft_prio->flow_table = ft;
2090
2091free:
2092	if (err)
2093		kfree(handler);
2094	kvfree(spec);
2095	return err ? ERR_PTR(err) : handler;
2096}
2097
2098static bool raw_fs_is_multicast(struct mlx5_ib_flow_matcher *fs_matcher,
2099				void *match_v)
2100{
2101	void *match_c;
2102	void *match_v_set_lyr_2_4, *match_c_set_lyr_2_4;
2103	void *dmac, *dmac_mask;
2104	void *ipv4, *ipv4_mask;
2105
2106	if (!(fs_matcher->match_criteria_enable &
2107	      (1 << MATCH_CRITERIA_ENABLE_OUTER_BIT)))
2108		return false;
2109
2110	match_c = fs_matcher->matcher_mask.match_params;
2111	match_v_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_v,
2112					   outer_headers);
2113	match_c_set_lyr_2_4 = MLX5_ADDR_OF(fte_match_param, match_c,
2114					   outer_headers);
2115
2116	dmac = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2117			    dmac_47_16);
2118	dmac_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2119				 dmac_47_16);
2120
2121	if (is_multicast_ether_addr(dmac) &&
2122	    is_multicast_ether_addr(dmac_mask))
2123		return true;
2124
2125	ipv4 = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_v_set_lyr_2_4,
2126			    dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2127
2128	ipv4_mask = MLX5_ADDR_OF(fte_match_set_lyr_2_4, match_c_set_lyr_2_4,
2129				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
2130
2131	if (ipv4_is_multicast(*(__be32 *)(ipv4)) &&
2132	    ipv4_is_multicast(*(__be32 *)(ipv4_mask)))
2133		return true;
2134
2135	return false;
2136}
2137
2138static struct mlx5_ib_flow_handler *raw_fs_rule_add(
2139	struct mlx5_ib_dev *dev, struct mlx5_ib_flow_matcher *fs_matcher,
2140	struct mlx5_flow_context *flow_context, struct mlx5_flow_act *flow_act,
2141	struct mlx5_fc *counter, void *cmd_in, int inlen, int dest_id, int dest_type)
2142{
2143	struct mlx5_flow_destination *dst;
2144	struct mlx5_ib_flow_prio *ft_prio;
2145	struct mlx5_ib_flow_handler *handler;
2146	int dst_num = 0;
2147	bool mcast;
2148	int err;
2149
2150	if (fs_matcher->flow_type != MLX5_IB_FLOW_TYPE_NORMAL)
2151		return ERR_PTR(-EOPNOTSUPP);
2152
2153	if (fs_matcher->priority > MLX5_IB_FLOW_LAST_PRIO)
2154		return ERR_PTR(-ENOMEM);
2155
2156	dst = kcalloc(2, sizeof(*dst), GFP_KERNEL);
2157	if (!dst)
2158		return ERR_PTR(-ENOMEM);
2159
2160	mcast = raw_fs_is_multicast(fs_matcher, cmd_in);
2161	mutex_lock(&dev->flow_db->lock);
2162
2163	ft_prio = _get_flow_table(dev, fs_matcher->priority,
2164				  fs_matcher->ns_type, mcast,
2165				  fs_matcher->ib_port);
2166	if (IS_ERR(ft_prio)) {
2167		err = PTR_ERR(ft_prio);
2168		goto unlock;
2169	}
2170
2171	switch (dest_type) {
2172	case MLX5_FLOW_DESTINATION_TYPE_TIR:
2173		dst[dst_num].type = dest_type;
2174		dst[dst_num++].tir_num = dest_id;
2175		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2176		break;
2177	case MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE:
2178		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM;
2179		dst[dst_num++].ft_num = dest_id;
2180		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2181		break;
2182	case MLX5_FLOW_DESTINATION_TYPE_PORT:
2183		dst[dst_num++].type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2184		flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW;
2185		break;
2186	default:
2187		break;
2188	}
2189
2190	if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
2191		if (WARN_ON(!counter)) {
2192			err = -EINVAL;
2193			goto unlock;
2194		}
2195		dst[dst_num].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
2196		dst[dst_num].counter = counter;
2197		dst_num++;
2198	}
2199
2200	handler = _create_raw_flow_rule(dev, ft_prio, dst_num ? dst : NULL,
2201					fs_matcher, flow_context, flow_act,
2202					cmd_in, inlen, dst_num);
2203
2204	if (IS_ERR(handler)) {
2205		err = PTR_ERR(handler);
2206		goto destroy_ft;
2207	}
2208
2209	mutex_unlock(&dev->flow_db->lock);
2210	atomic_inc(&fs_matcher->usecnt);
2211	handler->flow_matcher = fs_matcher;
2212
2213	kfree(dst);
2214
2215	return handler;
2216
2217destroy_ft:
2218	put_flow_table(dev, ft_prio, false);
2219unlock:
2220	mutex_unlock(&dev->flow_db->lock);
2221	kfree(dst);
2222
2223	return ERR_PTR(err);
2224}
2225
2226static void destroy_flow_action_raw(struct mlx5_ib_flow_action *maction)
2227{
2228	switch (maction->flow_action_raw.sub_type) {
2229	case MLX5_IB_FLOW_ACTION_MODIFY_HEADER:
2230		mlx5_modify_header_dealloc(maction->flow_action_raw.dev->mdev,
2231					   maction->flow_action_raw.modify_hdr);
2232		break;
2233	case MLX5_IB_FLOW_ACTION_PACKET_REFORMAT:
2234		mlx5_packet_reformat_dealloc(maction->flow_action_raw.dev->mdev,
2235					     maction->flow_action_raw.pkt_reformat);
2236		break;
2237	case MLX5_IB_FLOW_ACTION_DECAP:
2238		break;
2239	default:
2240		break;
2241	}
2242}
2243
2244static int mlx5_ib_destroy_flow_action(struct ib_flow_action *action)
2245{
2246	struct mlx5_ib_flow_action *maction = to_mflow_act(action);
2247
2248	switch (action->type) {
2249	case IB_FLOW_ACTION_UNSPECIFIED:
2250		destroy_flow_action_raw(maction);
2251		break;
2252	default:
2253		WARN_ON(true);
2254		break;
2255	}
2256
2257	kfree(maction);
2258	return 0;
2259}
2260
2261static int
2262mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
2263			     enum mlx5_flow_namespace_type *namespace)
2264{
2265	switch (table_type) {
2266	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX:
2267		*namespace = MLX5_FLOW_NAMESPACE_BYPASS;
2268		break;
2269	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
2270		*namespace = MLX5_FLOW_NAMESPACE_EGRESS;
2271		break;
2272	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
2273		*namespace = MLX5_FLOW_NAMESPACE_FDB_BYPASS;
2274		break;
2275	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX:
2276		*namespace = MLX5_FLOW_NAMESPACE_RDMA_RX;
2277		break;
2278	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX:
2279		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TX;
2280		break;
2281	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_RX:
2282		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX;
2283		break;
2284	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TRANSPORT_TX:
2285		*namespace = MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX;
2286		break;
2287	default:
2288		return -EINVAL;
2289	}
2290
2291	return 0;
2292}
2293
2294static const struct uverbs_attr_spec mlx5_ib_flow_type[] = {
2295	[MLX5_IB_FLOW_TYPE_NORMAL] = {
2296		.type = UVERBS_ATTR_TYPE_PTR_IN,
2297		.u.ptr = {
2298			.len = sizeof(u16), /* data is priority */
2299			.min_len = sizeof(u16),
2300		}
2301	},
2302	[MLX5_IB_FLOW_TYPE_SNIFFER] = {
2303		.type = UVERBS_ATTR_TYPE_PTR_IN,
2304		UVERBS_ATTR_NO_DATA(),
2305	},
2306	[MLX5_IB_FLOW_TYPE_ALL_DEFAULT] = {
2307		.type = UVERBS_ATTR_TYPE_PTR_IN,
2308		UVERBS_ATTR_NO_DATA(),
2309	},
2310	[MLX5_IB_FLOW_TYPE_MC_DEFAULT] = {
2311		.type = UVERBS_ATTR_TYPE_PTR_IN,
2312		UVERBS_ATTR_NO_DATA(),
2313	},
2314};
2315
2316static bool is_flow_dest(void *obj, int *dest_id, int *dest_type)
2317{
2318	struct devx_obj *devx_obj = obj;
2319	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2320
2321	switch (opcode) {
2322	case MLX5_CMD_OP_DESTROY_TIR:
2323		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2324		*dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
2325				    obj_id);
2326		return true;
2327
2328	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
2329		*dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2330		*dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
2331				    table_id);
2332		return true;
2333	default:
2334		return false;
2335	}
2336}
2337
2338static int get_dests(struct uverbs_attr_bundle *attrs,
2339		     struct mlx5_ib_flow_matcher *fs_matcher, int *dest_id,
2340		     int *dest_type, struct ib_qp **qp, u32 *flags)
2341{
2342	bool dest_devx, dest_qp;
2343	void *devx_obj;
2344	int err;
2345
2346	dest_devx = uverbs_attr_is_valid(attrs,
2347					 MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2348	dest_qp = uverbs_attr_is_valid(attrs,
2349				       MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2350
2351	*flags = 0;
2352	err = uverbs_get_flags32(flags, attrs, MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
2353				 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS |
2354					 MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP);
2355	if (err)
2356		return err;
2357
2358	/* Both flags are not allowed */
2359	if (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS &&
2360	    *flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2361		return -EINVAL;
2362
2363	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
2364		if (dest_devx && (dest_qp || *flags))
2365			return -EINVAL;
2366		else if (dest_qp && *flags)
2367			return -EINVAL;
2368	}
2369
2370	/* Allow only DEVX object, drop as dest for FDB */
2371	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2372	    !(dest_devx || (*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)))
2373		return -EINVAL;
2374
2375	/* Allow only DEVX object or QP as dest when inserting to RDMA_RX */
2376	if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2377	     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2378	    ((!dest_devx && !dest_qp) || (dest_devx && dest_qp)))
2379		return -EINVAL;
2380
2381	*qp = NULL;
2382	if (dest_devx) {
2383		devx_obj =
2384			uverbs_attr_get_obj(attrs,
2385					    MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
2386
2387		/* Verify that the given DEVX object is a flow
2388		 * steering destination.
2389		 */
2390		if (!is_flow_dest(devx_obj, dest_id, dest_type))
2391			return -EINVAL;
2392		/* Allow only flow table as dest when inserting to FDB or RDMA_RX */
2393		if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS ||
2394		     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_RX ||
2395		     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX) &&
2396		    *dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
2397			return -EINVAL;
2398	} else if (dest_qp) {
2399		struct mlx5_ib_qp *mqp;
2400
2401		*qp = uverbs_attr_get_obj(attrs,
2402					  MLX5_IB_ATTR_CREATE_FLOW_DEST_QP);
2403		if (IS_ERR(*qp))
2404			return PTR_ERR(*qp);
2405
2406		if ((*qp)->qp_type != IB_QPT_RAW_PACKET)
2407			return -EINVAL;
2408
2409		mqp = to_mqp(*qp);
2410		if (mqp->is_rss)
2411			*dest_id = mqp->rss_qp.tirn;
2412		else
2413			*dest_id = mqp->raw_packet_qp.rq.tirn;
2414		*dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
2415	} else if ((fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2416		    fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2417		    fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) &&
2418		   !(*flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)) {
2419		*dest_type = MLX5_FLOW_DESTINATION_TYPE_PORT;
2420	}
2421
2422	if (*dest_type == MLX5_FLOW_DESTINATION_TYPE_TIR &&
2423	    (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS ||
2424	     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TX ||
2425	     fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX))
2426		return -EINVAL;
2427
2428	return 0;
2429}
2430
2431static bool
2432is_flow_counter(void *obj, u32 offset, u32 *counter_id, u32 *fc_bulk_size)
2433{
2434	struct devx_obj *devx_obj = obj;
2435	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
2436
2437	if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
2438
2439		if (offset && offset >= devx_obj->flow_counter_bulk_size)
2440			return false;
2441
2442		*fc_bulk_size = devx_obj->flow_counter_bulk_size;
2443		*counter_id = MLX5_GET(dealloc_flow_counter_in,
2444				       devx_obj->dinbox,
2445				       flow_counter_id);
2446		*counter_id += offset;
2447		return true;
2448	}
2449
2450	return false;
2451}
2452
2453#define MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS 2
2454static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
2455	struct uverbs_attr_bundle *attrs)
2456{
2457	struct mlx5_flow_context flow_context = {.flow_tag =
2458		MLX5_FS_DEFAULT_FLOW_TAG};
2459	int dest_id, dest_type = -1, inlen, len, ret, i;
2460	struct mlx5_ib_flow_handler *flow_handler;
2461	struct mlx5_ib_flow_matcher *fs_matcher;
2462	struct ib_uobject **arr_flow_actions;
2463	struct ib_uflow_resources *uflow_res;
2464	struct mlx5_flow_act flow_act = {};
2465	struct mlx5_fc *counter = NULL;
2466	struct ib_qp *qp = NULL;
2467	void *devx_obj, *cmd_in;
2468	struct ib_uobject *uobj;
2469	struct mlx5_ib_dev *dev;
2470	u32 flags;
2471
2472	if (!rdma_uattrs_has_raw_cap(attrs))
2473		return -EPERM;
2474
2475	fs_matcher = uverbs_attr_get_obj(attrs,
2476					 MLX5_IB_ATTR_CREATE_FLOW_MATCHER);
2477	uobj =  uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
2478	dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2479
2480	if (get_dests(attrs, fs_matcher, &dest_id, &dest_type, &qp, &flags))
2481		return -EINVAL;
2482
2483	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DEFAULT_MISS)
2484		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS;
2485
2486	if (flags & MLX5_IB_ATTR_CREATE_FLOW_FLAGS_DROP)
2487		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
2488
2489	len = uverbs_attr_get_uobjs_arr(attrs,
2490		MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX, &arr_flow_actions);
2491	if (len) {
2492		u32 *offset_attr, fc_bulk_size, offset = 0, counter_id = 0;
2493		devx_obj = arr_flow_actions[0]->object;
2494
2495		if (uverbs_attr_is_valid(attrs,
2496					 MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET)) {
2497
2498			int num_offsets = uverbs_attr_ptr_get_array_size(
2499				attrs,
2500				MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
2501				sizeof(u32));
2502
2503			if (num_offsets != 1)
2504				return -EINVAL;
2505
2506			offset_attr = uverbs_attr_get_alloced_ptr(
2507				attrs,
2508				MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET);
2509			offset = *offset_attr;
2510		}
2511
2512		if (!is_flow_counter(devx_obj, offset, &counter_id, &fc_bulk_size))
2513			return -EINVAL;
2514		counter = mlx5_fc_local_create(counter_id, offset, fc_bulk_size);
2515		if (IS_ERR(counter))
2516			return PTR_ERR(counter);
2517
2518		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
2519	}
2520
2521	cmd_in = uverbs_attr_get_alloced_ptr(
2522		attrs, MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2523	inlen = uverbs_attr_get_len(attrs,
2524				    MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE);
2525
2526	uflow_res = flow_resources_alloc(MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS);
2527	if (!uflow_res) {
2528		ret = -ENOMEM;
2529		goto destroy_counter;
2530	}
2531
2532	len = uverbs_attr_get_uobjs_arr(attrs,
2533		MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS, &arr_flow_actions);
2534	for (i = 0; i < len; i++) {
2535		struct mlx5_ib_flow_action *maction =
2536			to_mflow_act(arr_flow_actions[i]->object);
2537
2538		ret = parse_flow_flow_action(maction, false, &flow_act);
2539		if (ret)
2540			goto err_out;
2541		flow_resources_add(uflow_res, IB_FLOW_SPEC_ACTION_HANDLE,
2542				   arr_flow_actions[i]->object);
2543	}
2544
2545	ret = uverbs_copy_from(&flow_context.flow_tag, attrs,
2546			       MLX5_IB_ATTR_CREATE_FLOW_TAG);
2547	if (!ret) {
2548		if (flow_context.flow_tag >= BIT(24)) {
2549			ret = -EINVAL;
2550			goto err_out;
2551		}
2552		flow_context.flags |= FLOW_CONTEXT_HAS_TAG;
2553	}
2554
2555	flow_handler =
2556		raw_fs_rule_add(dev, fs_matcher, &flow_context, &flow_act,
2557				counter, cmd_in, inlen, dest_id, dest_type);
2558	if (IS_ERR(flow_handler)) {
2559		ret = PTR_ERR(flow_handler);
2560		goto err_out;
2561	}
2562
2563	ib_set_flow(uobj, &flow_handler->ibflow, qp, &dev->ib_dev, uflow_res);
2564
2565	return 0;
2566err_out:
2567	ib_uverbs_flow_resources_free(uflow_res);
2568destroy_counter:
2569	if (counter)
2570		mlx5_fc_local_destroy(counter);
2571	return ret;
2572}
2573
2574static int flow_matcher_cleanup(struct ib_uobject *uobject,
2575				enum rdma_remove_reason why,
2576				struct uverbs_attr_bundle *attrs)
2577{
2578	struct mlx5_ib_flow_matcher *obj = uobject->object;
2579
2580	if (atomic_read(&obj->usecnt))
2581		return -EBUSY;
2582
2583	kfree(obj);
2584	return 0;
2585}
2586
2587static int steering_anchor_create_ft(struct mlx5_ib_dev *dev,
2588				     struct mlx5_ib_flow_prio *ft_prio,
2589				     enum mlx5_flow_namespace_type ns_type)
2590{
2591	struct mlx5_flow_table_attr ft_attr = {};
2592	struct mlx5_flow_namespace *ns;
2593	struct mlx5_flow_table *ft;
2594
2595	if (ft_prio->anchor.ft)
2596		return 0;
2597
2598	ns = mlx5_get_flow_namespace(dev->mdev, ns_type);
2599	if (!ns)
2600		return -EOPNOTSUPP;
2601
2602	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
2603	ft_attr.uid = MLX5_SHARED_RESOURCE_UID;
2604	ft_attr.prio = 0;
2605	ft_attr.max_fte = 2;
2606	ft_attr.level = 1;
2607
2608	ft = mlx5_create_flow_table(ns, &ft_attr);
2609	if (IS_ERR(ft))
2610		return PTR_ERR(ft);
2611
2612	ft_prio->anchor.ft = ft;
2613
2614	return 0;
2615}
2616
2617static void steering_anchor_destroy_ft(struct mlx5_ib_flow_prio *ft_prio)
2618{
2619	if (ft_prio->anchor.ft) {
2620		mlx5_destroy_flow_table(ft_prio->anchor.ft);
2621		ft_prio->anchor.ft = NULL;
2622	}
2623}
2624
2625static int
2626steering_anchor_create_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2627{
2628	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2629	struct mlx5_flow_group *fg;
2630	void *flow_group_in;
2631	int err = 0;
2632
2633	if (ft_prio->anchor.fg_drop)
2634		return 0;
2635
2636	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2637	if (!flow_group_in)
2638		return -ENOMEM;
2639
2640	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
2641	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
2642
2643	fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2644	if (IS_ERR(fg)) {
2645		err = PTR_ERR(fg);
2646		goto out;
2647	}
2648
2649	ft_prio->anchor.fg_drop = fg;
2650
2651out:
2652	kvfree(flow_group_in);
2653
2654	return err;
2655}
2656
2657static void
2658steering_anchor_destroy_fg_drop(struct mlx5_ib_flow_prio *ft_prio)
2659{
2660	if (ft_prio->anchor.fg_drop) {
2661		mlx5_destroy_flow_group(ft_prio->anchor.fg_drop);
2662		ft_prio->anchor.fg_drop = NULL;
2663	}
2664}
2665
2666static int
2667steering_anchor_create_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2668{
2669	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2670	struct mlx5_flow_group *fg;
2671	void *flow_group_in;
2672	int err = 0;
2673
2674	if (ft_prio->anchor.fg_goto_table)
2675		return 0;
2676
2677	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2678	if (!flow_group_in)
2679		return -ENOMEM;
2680
2681	fg = mlx5_create_flow_group(ft_prio->anchor.ft, flow_group_in);
2682	if (IS_ERR(fg)) {
2683		err = PTR_ERR(fg);
2684		goto out;
2685	}
2686	ft_prio->anchor.fg_goto_table = fg;
2687
2688out:
2689	kvfree(flow_group_in);
2690
2691	return err;
2692}
2693
2694static void
2695steering_anchor_destroy_fg_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2696{
2697	if (ft_prio->anchor.fg_goto_table) {
2698		mlx5_destroy_flow_group(ft_prio->anchor.fg_goto_table);
2699		ft_prio->anchor.fg_goto_table = NULL;
2700	}
2701}
2702
2703static int
2704steering_anchor_create_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2705{
2706	struct mlx5_flow_act flow_act = {};
2707	struct mlx5_flow_handle *handle;
2708
2709	if (ft_prio->anchor.rule_drop)
2710		return 0;
2711
2712	flow_act.fg = ft_prio->anchor.fg_drop;
2713	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
2714
2715	handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2716				     NULL, 0);
2717	if (IS_ERR(handle))
2718		return PTR_ERR(handle);
2719
2720	ft_prio->anchor.rule_drop = handle;
2721
2722	return 0;
2723}
2724
2725static void steering_anchor_destroy_rule_drop(struct mlx5_ib_flow_prio *ft_prio)
2726{
2727	if (ft_prio->anchor.rule_drop) {
2728		mlx5_del_flow_rules(ft_prio->anchor.rule_drop);
2729		ft_prio->anchor.rule_drop = NULL;
2730	}
2731}
2732
2733static int
2734steering_anchor_create_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2735{
2736	struct mlx5_flow_destination dest = {};
2737	struct mlx5_flow_act flow_act = {};
2738	struct mlx5_flow_handle *handle;
2739
2740	if (ft_prio->anchor.rule_goto_table)
2741		return 0;
2742
2743	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2744	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
2745	flow_act.fg = ft_prio->anchor.fg_goto_table;
2746
2747	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2748	dest.ft = ft_prio->flow_table;
2749
2750	handle = mlx5_add_flow_rules(ft_prio->anchor.ft, NULL, &flow_act,
2751				     &dest, 1);
2752	if (IS_ERR(handle))
2753		return PTR_ERR(handle);
2754
2755	ft_prio->anchor.rule_goto_table = handle;
2756
2757	return 0;
2758}
2759
2760static void
2761steering_anchor_destroy_rule_goto_table(struct mlx5_ib_flow_prio *ft_prio)
2762{
2763	if (ft_prio->anchor.rule_goto_table) {
2764		mlx5_del_flow_rules(ft_prio->anchor.rule_goto_table);
2765		ft_prio->anchor.rule_goto_table = NULL;
2766	}
2767}
2768
2769static int steering_anchor_create_res(struct mlx5_ib_dev *dev,
2770				      struct mlx5_ib_flow_prio *ft_prio,
2771				      enum mlx5_flow_namespace_type ns_type)
2772{
2773	int err;
2774
2775	err = steering_anchor_create_ft(dev, ft_prio, ns_type);
2776	if (err)
2777		return err;
2778
2779	err = steering_anchor_create_fg_drop(ft_prio);
2780	if (err)
2781		goto destroy_ft;
2782
2783	err = steering_anchor_create_fg_goto_table(ft_prio);
2784	if (err)
2785		goto destroy_fg_drop;
2786
2787	err = steering_anchor_create_rule_drop(ft_prio);
2788	if (err)
2789		goto destroy_fg_goto_table;
2790
2791	err = steering_anchor_create_rule_goto_table(ft_prio);
2792	if (err)
2793		goto destroy_rule_drop;
2794
2795	return 0;
2796
2797destroy_rule_drop:
2798	steering_anchor_destroy_rule_drop(ft_prio);
2799destroy_fg_goto_table:
2800	steering_anchor_destroy_fg_goto_table(ft_prio);
2801destroy_fg_drop:
2802	steering_anchor_destroy_fg_drop(ft_prio);
2803destroy_ft:
2804	steering_anchor_destroy_ft(ft_prio);
2805
2806	return err;
2807}
2808
2809static void mlx5_steering_anchor_destroy_res(struct mlx5_ib_flow_prio *ft_prio)
2810{
2811	steering_anchor_destroy_rule_goto_table(ft_prio);
2812	steering_anchor_destroy_rule_drop(ft_prio);
2813	steering_anchor_destroy_fg_goto_table(ft_prio);
2814	steering_anchor_destroy_fg_drop(ft_prio);
2815	steering_anchor_destroy_ft(ft_prio);
2816}
2817
2818static int steering_anchor_cleanup(struct ib_uobject *uobject,
2819				   enum rdma_remove_reason why,
2820				   struct uverbs_attr_bundle *attrs)
2821{
2822	struct mlx5_ib_steering_anchor *obj = uobject->object;
2823
2824	if (atomic_read(&obj->usecnt))
2825		return -EBUSY;
2826
2827	mutex_lock(&obj->dev->flow_db->lock);
2828	if (!--obj->ft_prio->anchor.rule_goto_table_ref)
2829		steering_anchor_destroy_rule_goto_table(obj->ft_prio);
2830
2831	put_flow_table(obj->dev, obj->ft_prio, true);
2832	mutex_unlock(&obj->dev->flow_db->lock);
2833
2834	kfree(obj);
2835	return 0;
2836}
2837
2838static void fs_cleanup_anchor(struct mlx5_ib_flow_prio *prio,
2839			      int count)
2840{
2841	while (count--)
2842		mlx5_steering_anchor_destroy_res(&prio[count]);
2843}
2844
2845void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev)
2846{
2847	fs_cleanup_anchor(dev->flow_db->prios, MLX5_IB_NUM_FLOW_FT);
2848	fs_cleanup_anchor(dev->flow_db->egress_prios, MLX5_IB_NUM_FLOW_FT);
2849	fs_cleanup_anchor(dev->flow_db->sniffer, MLX5_IB_NUM_SNIFFER_FTS);
2850	fs_cleanup_anchor(dev->flow_db->egress, MLX5_IB_NUM_EGRESS_FTS);
2851	fs_cleanup_anchor(dev->flow_db->fdb, MLX5_IB_NUM_FDB_FTS);
2852	fs_cleanup_anchor(dev->flow_db->rdma_rx, MLX5_IB_NUM_FLOW_FT);
2853	fs_cleanup_anchor(dev->flow_db->rdma_tx, MLX5_IB_NUM_FLOW_FT);
2854}
2855
2856static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
2857			      struct mlx5_ib_flow_matcher *obj)
2858{
2859	enum mlx5_ib_uapi_flow_table_type ft_type =
2860		MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
2861	u32 flags;
2862	int err;
2863
2864	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
2865	 * users should switch to it. We leave this to not break userspace
2866	 */
2867	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
2868	    uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
2869		return -EINVAL;
2870
2871	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
2872		err = uverbs_get_const(&ft_type, attrs,
2873				       MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
2874		if (err)
2875			return err;
2876
2877		err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
2878		if (err)
2879			return err;
2880
2881		return 0;
2882	}
2883
2884	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
2885		err = uverbs_get_flags32(&flags, attrs,
2886					 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
2887					 IB_FLOW_ATTR_FLAGS_EGRESS);
2888		if (err)
2889			return err;
2890
2891		if (flags)
2892			return mlx5_ib_ft_type_to_namespace(
2893				MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
2894				&obj->ns_type);
2895	}
2896
2897	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
2898
2899	return 0;
2900}
2901
2902static bool verify_context_caps(struct mlx5_ib_dev *dev, u64 enabled_caps)
2903{
2904	if (is_mdev_switchdev_mode(dev->mdev))
2905		return UCAP_ENABLED(enabled_caps,
2906				    RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
2907
2908	return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL);
2909}
2910
2911static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
2912	struct uverbs_attr_bundle *attrs)
2913{
2914	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2915		attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
2916	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2917	struct mlx5_ib_flow_matcher *obj;
2918	int err;
2919
2920	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
2921	if (!obj)
2922		return -ENOMEM;
2923
2924	obj->mask_len = uverbs_attr_get_len(
2925		attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2926	err = uverbs_copy_from(&obj->matcher_mask,
2927			       attrs,
2928			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
2929	if (err)
2930		goto end;
2931
2932	obj->flow_type = uverbs_attr_get_enum_id(
2933		attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2934
2935	if (obj->flow_type == MLX5_IB_FLOW_TYPE_NORMAL) {
2936		err = uverbs_copy_from(&obj->priority,
2937				       attrs,
2938				       MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE);
2939		if (err)
2940			goto end;
2941	}
2942
2943	err = uverbs_copy_from(&obj->match_criteria_enable,
2944			       attrs,
2945			       MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA);
2946	if (err)
2947		goto end;
2948
2949	err = mlx5_ib_matcher_ns(attrs, obj);
2950	if (err)
2951		goto end;
2952
2953	if (obj->ns_type == MLX5_FLOW_NAMESPACE_FDB_BYPASS &&
2954	    mlx5_eswitch_mode(dev->mdev) != MLX5_ESWITCH_OFFLOADS) {
2955		err = -EINVAL;
2956		goto end;
2957	}
2958
2959	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT)) {
2960		err = uverbs_copy_from(&obj->ib_port, attrs,
2961				       MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT);
2962		if (err)
2963			goto end;
2964		if (!rdma_is_port_valid(&dev->ib_dev, obj->ib_port)) {
2965			err = -EINVAL;
2966			goto end;
2967		}
2968		if (obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX &&
2969		    obj->ns_type != MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX) {
2970			err = -EINVAL;
2971			goto end;
2972		}
2973		if (!verify_context_caps(dev, uobj->context->enabled_caps)) {
2974			err = -EOPNOTSUPP;
2975			goto end;
2976		}
2977	}
2978
2979	uobj->object = obj;
2980	obj->mdev = dev->mdev;
2981	atomic_set(&obj->usecnt, 0);
2982	return 0;
2983
2984end:
2985	kfree(obj);
2986	return err;
2987}
2988
2989static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
2990	struct uverbs_attr_bundle *attrs)
2991{
2992	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2993		attrs, MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE);
2994	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
2995	enum mlx5_ib_uapi_flow_table_type ib_uapi_ft_type;
2996	enum mlx5_flow_namespace_type ns_type;
2997	struct mlx5_ib_steering_anchor *obj;
2998	struct mlx5_ib_flow_prio *ft_prio;
2999	u16 priority;
3000	u32 ft_id;
3001	int err;
3002
3003	if (!rdma_dev_has_raw_cap(&dev->ib_dev))
3004		return -EPERM;
3005
3006	err = uverbs_get_const(&ib_uapi_ft_type, attrs,
3007			       MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE);
3008	if (err)
3009		return err;
3010
3011	err = mlx5_ib_ft_type_to_namespace(ib_uapi_ft_type, &ns_type);
3012	if (err)
3013		return err;
3014
3015	err = uverbs_copy_from(&priority, attrs,
3016			       MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY);
3017	if (err)
3018		return err;
3019
3020	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3021	if (!obj)
3022		return -ENOMEM;
3023
3024	mutex_lock(&dev->flow_db->lock);
3025
3026	ft_prio = _get_flow_table(dev, priority, ns_type, 0, 0);
3027	if (IS_ERR(ft_prio)) {
3028		err = PTR_ERR(ft_prio);
3029		goto free_obj;
3030	}
3031
3032	ft_prio->refcount++;
3033
3034	if (!ft_prio->anchor.rule_goto_table_ref) {
3035		err = steering_anchor_create_res(dev, ft_prio, ns_type);
3036		if (err)
3037			goto put_flow_table;
3038	}
3039
3040	ft_prio->anchor.rule_goto_table_ref++;
3041
3042	ft_id = mlx5_flow_table_id(ft_prio->anchor.ft);
3043
3044	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3045			     &ft_id, sizeof(ft_id));
3046	if (err)
3047		goto destroy_res;
3048
3049	mutex_unlock(&dev->flow_db->lock);
3050
3051	uobj->object = obj;
3052	obj->dev = dev;
3053	obj->ft_prio = ft_prio;
3054	atomic_set(&obj->usecnt, 0);
3055
3056	return 0;
3057
3058destroy_res:
3059	--ft_prio->anchor.rule_goto_table_ref;
3060	mlx5_steering_anchor_destroy_res(ft_prio);
3061put_flow_table:
3062	put_flow_table(dev, ft_prio, true);
3063free_obj:
3064	mutex_unlock(&dev->flow_db->lock);
3065	kfree(obj);
3066
3067	return err;
3068}
3069
3070static struct ib_flow_action *
3071mlx5_ib_create_modify_header(struct mlx5_ib_dev *dev,
3072			     enum mlx5_ib_uapi_flow_table_type ft_type,
3073			     u8 num_actions, void *in)
3074{
3075	enum mlx5_flow_namespace_type namespace;
3076	struct mlx5_ib_flow_action *maction;
3077	int ret;
3078
3079	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3080	if (ret)
3081		return ERR_PTR(-EINVAL);
3082
3083	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3084	if (!maction)
3085		return ERR_PTR(-ENOMEM);
3086
3087	maction->flow_action_raw.modify_hdr =
3088		mlx5_modify_header_alloc(dev->mdev, namespace, num_actions, in);
3089
3090	if (IS_ERR(maction->flow_action_raw.modify_hdr)) {
3091		ret = PTR_ERR(maction->flow_action_raw.modify_hdr);
3092		kfree(maction);
3093		return ERR_PTR(ret);
3094	}
3095	maction->flow_action_raw.sub_type =
3096		MLX5_IB_FLOW_ACTION_MODIFY_HEADER;
3097	maction->flow_action_raw.dev = dev;
3098
3099	return &maction->ib_action;
3100}
3101
3102static bool mlx5_ib_modify_header_supported(struct mlx5_ib_dev *dev)
3103{
3104	return MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
3105					 max_modify_header_actions) ||
3106	       MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
3107					 max_modify_header_actions) ||
3108	       MLX5_CAP_FLOWTABLE_RDMA_TX(dev->mdev,
3109					 max_modify_header_actions);
3110}
3111
3112static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
3113	struct uverbs_attr_bundle *attrs)
3114{
3115	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3116		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
3117	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3118	enum mlx5_ib_uapi_flow_table_type ft_type;
3119	struct ib_flow_action *action;
3120	int num_actions;
3121	void *in;
3122	int ret;
3123
3124	if (!mlx5_ib_modify_header_supported(mdev))
3125		return -EOPNOTSUPP;
3126
3127	in = uverbs_attr_get_alloced_ptr(attrs,
3128		MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM);
3129
3130	num_actions = uverbs_attr_ptr_get_array_size(
3131		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3132		MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto));
3133	if (num_actions < 0)
3134		return num_actions;
3135
3136	ret = uverbs_get_const(&ft_type, attrs,
3137			       MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE);
3138	if (ret)
3139		return ret;
3140	action = mlx5_ib_create_modify_header(mdev, ft_type, num_actions, in);
3141	if (IS_ERR(action))
3142		return PTR_ERR(action);
3143
3144	uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
3145				       IB_FLOW_ACTION_UNSPECIFIED);
3146
3147	return 0;
3148}
3149
3150static bool mlx5_ib_flow_action_packet_reformat_valid(struct mlx5_ib_dev *ibdev,
3151						      u8 packet_reformat_type,
3152						      u8 ft_type)
3153{
3154	switch (packet_reformat_type) {
3155	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3156		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3157			return MLX5_CAP_FLOWTABLE(ibdev->mdev,
3158						  encap_general_header);
3159		break;
3160	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3161		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX)
3162			return MLX5_CAP_FLOWTABLE_NIC_TX(ibdev->mdev,
3163				reformat_l2_to_l3_tunnel);
3164		break;
3165	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3166		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3167			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev,
3168				reformat_l3_tunnel_to_l2);
3169		break;
3170	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2:
3171		if (ft_type == MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX)
3172			return MLX5_CAP_FLOWTABLE_NIC_RX(ibdev->mdev, decap);
3173		break;
3174	default:
3175		break;
3176	}
3177
3178	return false;
3179}
3180
3181static int mlx5_ib_dv_to_prm_packet_reforamt_type(u8 dv_prt, u8 *prm_prt)
3182{
3183	switch (dv_prt) {
3184	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL:
3185		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL;
3186		break;
3187	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2:
3188		*prm_prt = MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2;
3189		break;
3190	case MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL:
3191		*prm_prt = MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL;
3192		break;
3193	default:
3194		return -EINVAL;
3195	}
3196
3197	return 0;
3198}
3199
3200static int mlx5_ib_flow_action_create_packet_reformat_ctx(
3201	struct mlx5_ib_dev *dev,
3202	struct mlx5_ib_flow_action *maction,
3203	u8 ft_type, u8 dv_prt,
3204	void *in, size_t len)
3205{
3206	struct mlx5_pkt_reformat_params reformat_params;
3207	enum mlx5_flow_namespace_type namespace;
3208	u8 prm_prt;
3209	int ret;
3210
3211	ret = mlx5_ib_ft_type_to_namespace(ft_type, &namespace);
3212	if (ret)
3213		return ret;
3214
3215	ret = mlx5_ib_dv_to_prm_packet_reforamt_type(dv_prt, &prm_prt);
3216	if (ret)
3217		return ret;
3218
3219	memset(&reformat_params, 0, sizeof(reformat_params));
3220	reformat_params.type = prm_prt;
3221	reformat_params.size = len;
3222	reformat_params.data = in;
3223	maction->flow_action_raw.pkt_reformat =
3224		mlx5_packet_reformat_alloc(dev->mdev, &reformat_params,
3225					   namespace);
3226	if (IS_ERR(maction->flow_action_raw.pkt_reformat)) {
3227		ret = PTR_ERR(maction->flow_action_raw.pkt_reformat);
3228		return ret;
3229	}
3230
3231	maction->flow_action_raw.sub_type =
3232		MLX5_IB_FLOW_ACTION_PACKET_REFORMAT;
3233	maction->flow_action_raw.dev = dev;
3234
3235	return 0;
3236}
3237
3238static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
3239	struct uverbs_attr_bundle *attrs)
3240{
3241	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
3242		MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
3243	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
3244	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
3245	enum mlx5_ib_uapi_flow_table_type ft_type;
3246	struct mlx5_ib_flow_action *maction;
3247	int ret;
3248
3249	ret = uverbs_get_const(&ft_type, attrs,
3250			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE);
3251	if (ret)
3252		return ret;
3253
3254	ret = uverbs_get_const(&dv_prt, attrs,
3255			       MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE);
3256	if (ret)
3257		return ret;
3258
3259	if (!mlx5_ib_flow_action_packet_reformat_valid(mdev, dv_prt, ft_type))
3260		return -EOPNOTSUPP;
3261
3262	maction = kzalloc(sizeof(*maction), GFP_KERNEL);
3263	if (!maction)
3264		return -ENOMEM;
3265
3266	if (dv_prt ==
3267	    MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2) {
3268		maction->flow_action_raw.sub_type =
3269			MLX5_IB_FLOW_ACTION_DECAP;
3270		maction->flow_action_raw.dev = mdev;
3271	} else {
3272		void *in;
3273		int len;
3274
3275		in = uverbs_attr_get_alloced_ptr(attrs,
3276			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3277		if (IS_ERR(in)) {
3278			ret = PTR_ERR(in);
3279			goto free_maction;
3280		}
3281
3282		len = uverbs_attr_get_len(attrs,
3283			MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF);
3284
3285		ret = mlx5_ib_flow_action_create_packet_reformat_ctx(mdev,
3286			maction, ft_type, dv_prt, in, len);
3287		if (ret)
3288			goto free_maction;
3289	}
3290
3291	uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
3292				       IB_FLOW_ACTION_UNSPECIFIED);
3293	return 0;
3294
3295free_maction:
3296	kfree(maction);
3297	return ret;
3298}
3299
3300DECLARE_UVERBS_NAMED_METHOD(
3301	MLX5_IB_METHOD_CREATE_FLOW,
3302	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3303			UVERBS_OBJECT_FLOW,
3304			UVERBS_ACCESS_NEW,
3305			UA_MANDATORY),
3306	UVERBS_ATTR_PTR_IN(
3307		MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
3308		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3309		UA_MANDATORY,
3310		UA_ALLOC_AND_COPY),
3311	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
3312			MLX5_IB_OBJECT_FLOW_MATCHER,
3313			UVERBS_ACCESS_READ,
3314			UA_MANDATORY),
3315	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
3316			UVERBS_OBJECT_QP,
3317			UVERBS_ACCESS_READ),
3318	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
3319			MLX5_IB_OBJECT_DEVX_OBJ,
3320			UVERBS_ACCESS_READ),
3321	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
3322			     UVERBS_OBJECT_FLOW_ACTION,
3323			     UVERBS_ACCESS_READ, 1,
3324			     MLX5_IB_CREATE_FLOW_MAX_FLOW_ACTIONS,
3325			     UA_OPTIONAL),
3326	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_TAG,
3327			   UVERBS_ATTR_TYPE(u32),
3328			   UA_OPTIONAL),
3329	UVERBS_ATTR_IDRS_ARR(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
3330			     MLX5_IB_OBJECT_DEVX_OBJ,
3331			     UVERBS_ACCESS_READ, 1, 1,
3332			     UA_OPTIONAL),
3333	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
3334			   UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
3335			   UA_OPTIONAL,
3336			   UA_ALLOC_AND_COPY),
3337	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_CREATE_FLOW_FLAGS,
3338			     enum mlx5_ib_create_flow_flags,
3339			     UA_OPTIONAL));
3340
3341DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3342	MLX5_IB_METHOD_DESTROY_FLOW,
3343	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_FLOW_HANDLE,
3344			UVERBS_OBJECT_FLOW,
3345			UVERBS_ACCESS_DESTROY,
3346			UA_MANDATORY));
3347
3348ADD_UVERBS_METHODS(mlx5_ib_fs,
3349		   UVERBS_OBJECT_FLOW,
3350		   &UVERBS_METHOD(MLX5_IB_METHOD_CREATE_FLOW),
3351		   &UVERBS_METHOD(MLX5_IB_METHOD_DESTROY_FLOW));
3352
3353DECLARE_UVERBS_NAMED_METHOD(
3354	MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER,
3355	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE,
3356			UVERBS_OBJECT_FLOW_ACTION,
3357			UVERBS_ACCESS_NEW,
3358			UA_MANDATORY),
3359	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
3360			   UVERBS_ATTR_MIN_SIZE(MLX5_UN_SZ_BYTES(
3361				   set_add_copy_action_in_auto)),
3362			   UA_MANDATORY,
3363			   UA_ALLOC_AND_COPY),
3364	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
3365			     enum mlx5_ib_uapi_flow_table_type,
3366			     UA_MANDATORY));
3367
3368DECLARE_UVERBS_NAMED_METHOD(
3369	MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
3370	UVERBS_ATTR_IDR(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE,
3371			UVERBS_OBJECT_FLOW_ACTION,
3372			UVERBS_ACCESS_NEW,
3373			UA_MANDATORY),
3374	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
3375			   UVERBS_ATTR_MIN_SIZE(1),
3376			   UA_ALLOC_AND_COPY,
3377			   UA_OPTIONAL),
3378	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
3379			     enum mlx5_ib_uapi_flow_action_packet_reformat_type,
3380			     UA_MANDATORY),
3381	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
3382			     enum mlx5_ib_uapi_flow_table_type,
3383			     UA_MANDATORY));
3384
3385ADD_UVERBS_METHODS(
3386	mlx5_ib_flow_actions,
3387	UVERBS_OBJECT_FLOW_ACTION,
3388	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER),
3389	&UVERBS_METHOD(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT));
3390
3391DECLARE_UVERBS_NAMED_METHOD(
3392	MLX5_IB_METHOD_FLOW_MATCHER_CREATE,
3393	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE,
3394			MLX5_IB_OBJECT_FLOW_MATCHER,
3395			UVERBS_ACCESS_NEW,
3396			UA_MANDATORY),
3397	UVERBS_ATTR_PTR_IN(
3398		MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
3399		UVERBS_ATTR_SIZE(1, sizeof(struct mlx5_ib_match_params)),
3400		UA_MANDATORY),
3401	UVERBS_ATTR_ENUM_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
3402			    mlx5_ib_flow_type,
3403			    UA_MANDATORY),
3404	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
3405			   UVERBS_ATTR_TYPE(u8),
3406			   UA_MANDATORY),
3407	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
3408			     enum ib_flow_flags,
3409			     UA_OPTIONAL),
3410	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
3411			     enum mlx5_ib_uapi_flow_table_type,
3412			     UA_OPTIONAL),
3413	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_FLOW_MATCHER_IB_PORT,
3414			   UVERBS_ATTR_TYPE(u32),
3415			   UA_OPTIONAL));
3416
3417DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3418	MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
3419	UVERBS_ATTR_IDR(MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE,
3420			MLX5_IB_OBJECT_FLOW_MATCHER,
3421			UVERBS_ACCESS_DESTROY,
3422			UA_MANDATORY));
3423
3424DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
3425			    UVERBS_TYPE_ALLOC_IDR(flow_matcher_cleanup),
3426			    &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
3427			    &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
3428
3429DECLARE_UVERBS_NAMED_METHOD(
3430	MLX5_IB_METHOD_STEERING_ANCHOR_CREATE,
3431	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_CREATE_HANDLE,
3432			MLX5_IB_OBJECT_STEERING_ANCHOR,
3433			UVERBS_ACCESS_NEW,
3434			UA_MANDATORY),
3435	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_TYPE,
3436			     enum mlx5_ib_uapi_flow_table_type,
3437			     UA_MANDATORY),
3438	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_PRIORITY,
3439			   UVERBS_ATTR_TYPE(u16),
3440			   UA_MANDATORY),
3441	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_STEERING_ANCHOR_FT_ID,
3442			   UVERBS_ATTR_TYPE(u32),
3443			   UA_MANDATORY));
3444
3445DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3446	MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY,
3447	UVERBS_ATTR_IDR(MLX5_IB_ATTR_STEERING_ANCHOR_DESTROY_HANDLE,
3448			MLX5_IB_OBJECT_STEERING_ANCHOR,
3449			UVERBS_ACCESS_DESTROY,
3450			UA_MANDATORY));
3451
3452DECLARE_UVERBS_NAMED_OBJECT(
3453	MLX5_IB_OBJECT_STEERING_ANCHOR,
3454	UVERBS_TYPE_ALLOC_IDR(steering_anchor_cleanup),
3455	&UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE),
3456	&UVERBS_METHOD(MLX5_IB_METHOD_STEERING_ANCHOR_DESTROY));
3457
3458const struct uapi_definition mlx5_ib_flow_defs[] = {
3459	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3460		MLX5_IB_OBJECT_FLOW_MATCHER),
3461	UAPI_DEF_CHAIN_OBJ_TREE(
3462		UVERBS_OBJECT_FLOW,
3463		&mlx5_ib_fs),
3464	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_FLOW_ACTION,
3465				&mlx5_ib_flow_actions),
3466	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3467		MLX5_IB_OBJECT_STEERING_ANCHOR,
3468		UAPI_DEF_IS_OBJ_SUPPORTED(mlx5_ib_shared_ft_allowed)),
3469	{},
3470};
3471
3472static const struct ib_device_ops flow_ops = {
3473	.create_flow = mlx5_ib_create_flow,
3474	.destroy_flow = mlx5_ib_destroy_flow,
3475	.destroy_flow_action = mlx5_ib_destroy_flow_action,
3476};
3477
3478int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
3479{
3480	int i, j;
3481
3482	dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
3483
3484	if (!dev->flow_db)
3485		return -ENOMEM;
3486
3487	for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {
3488		dev->flow_db->rdma_transport_rx[i] =
3489			kcalloc(dev->num_ports,
3490				sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3491		if (!dev->flow_db->rdma_transport_rx[i])
3492			goto free_rdma_transport_rx;
3493	}
3494
3495	for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {
3496		dev->flow_db->rdma_transport_tx[j] =
3497			kcalloc(dev->num_ports,
3498				sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
3499		if (!dev->flow_db->rdma_transport_tx[j])
3500			goto free_rdma_transport_tx;
3501	}
3502
3503	mutex_init(&dev->flow_db->lock);
3504
3505	ib_set_device_ops(&dev->ib_dev, &flow_ops);
3506	return 0;
3507
3508free_rdma_transport_tx:
3509	while (j--)
3510		kfree(dev->flow_db->rdma_transport_tx[j]);
3511free_rdma_transport_rx:
3512	while (i--)
3513		kfree(dev->flow_db->rdma_transport_rx[i]);
3514	kfree(dev->flow_db);
3515	return -ENOMEM;
3516}