bridge.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2021 Mellanox Technologies. */
   3
   4#include <linux/build_bug.h>
   5#include <linux/list.h>
   6#include <linux/notifier.h>
   7#include <net/netevent.h>
   8#include <net/switchdev.h>
   9#include "lib/devcom.h"
  10#include "bridge.h"
  11#include "eswitch.h"
  12#include "bridge_priv.h"
  13#define CREATE_TRACE_POINTS
  14#include "diag/bridge_tracepoint.h"
  15
  16static const struct rhashtable_params fdb_ht_params = {
  17	.key_offset = offsetof(struct mlx5_esw_bridge_fdb_entry, key),
  18	.key_len = sizeof(struct mlx5_esw_bridge_fdb_key),
  19	.head_offset = offsetof(struct mlx5_esw_bridge_fdb_entry, ht_node),
  20	.automatic_shrinking = true,
  21};
  22
  23static void
  24mlx5_esw_bridge_fdb_offload_notify(struct net_device *dev, const unsigned char *addr, u16 vid,
  25				   unsigned long val)
  26{
  27	struct switchdev_notifier_fdb_info send_info = {};
  28
  29	send_info.addr = addr;
  30	send_info.vid = vid;
  31	send_info.offloaded = true;
  32	call_switchdev_notifiers(val, dev, &send_info.info, NULL);
  33}
  34
  35static void
  36mlx5_esw_bridge_fdb_del_notify(struct mlx5_esw_bridge_fdb_entry *entry)
  37{
  38	if (!(entry->flags & (MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER | MLX5_ESW_BRIDGE_FLAG_PEER)))
  39		mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr,
  40						   entry->key.vid,
  41						   SWITCHDEV_FDB_DEL_TO_BRIDGE);
  42}
  43
  44static bool mlx5_esw_bridge_pkt_reformat_vlan_pop_supported(struct mlx5_eswitch *esw)
  45{
  46	return BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat_remove)) &&
  47		MLX5_CAP_GEN_2(esw->dev, max_reformat_remove_size) >= sizeof(struct vlan_hdr) &&
  48		MLX5_CAP_GEN_2(esw->dev, max_reformat_remove_offset) >=
  49		offsetof(struct vlan_ethhdr, h_vlan_proto);
  50}
  51
  52static struct mlx5_pkt_reformat *
  53mlx5_esw_bridge_pkt_reformat_vlan_pop_create(struct mlx5_eswitch *esw)
  54{
  55	struct mlx5_pkt_reformat_params reformat_params = {};
  56
  57	reformat_params.type = MLX5_REFORMAT_TYPE_REMOVE_HDR;
  58	reformat_params.param_0 = MLX5_REFORMAT_CONTEXT_ANCHOR_MAC_START;
  59	reformat_params.param_1 = offsetof(struct vlan_ethhdr, h_vlan_proto);
  60	reformat_params.size = sizeof(struct vlan_hdr);
  61	return mlx5_packet_reformat_alloc(esw->dev, &reformat_params, MLX5_FLOW_NAMESPACE_FDB);
  62}
  63
  64struct mlx5_flow_table *
  65mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw)
  66{
  67	struct mlx5_flow_table_attr ft_attr = {};
  68	struct mlx5_core_dev *dev = esw->dev;
  69	struct mlx5_flow_namespace *ns;
  70	struct mlx5_flow_table *fdb;
  71
  72	ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
  73	if (!ns) {
  74		esw_warn(dev, "Failed to get FDB namespace\n");
  75		return ERR_PTR(-ENOENT);
  76	}
  77
  78	ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
  79	ft_attr.max_fte = max_fte;
  80	ft_attr.level = level;
  81	ft_attr.prio = FDB_BR_OFFLOAD;
  82	fdb = mlx5_create_flow_table(ns, &ft_attr);
  83	if (IS_ERR(fdb))
  84		esw_warn(dev, "Failed to create bridge FDB Table (err=%pe)\n",
  85			 fdb);
  86
  87	return fdb;
  88}
  89
  90static struct mlx5_flow_group *
  91mlx5_esw_bridge_ingress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto,
  92					     struct mlx5_eswitch *esw,
  93					     struct mlx5_flow_table *ingress_ft)
  94{
  95	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
  96	struct mlx5_flow_group *fg;
  97	u32 *in, *match;
  98
  99	in = kvzalloc(inlen, GFP_KERNEL);
 100	if (!in)
 101		return ERR_PTR(-ENOMEM);
 102
 103	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
 104		 MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2);
 105	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 106
 107	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16);
 108	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0);
 109	if (vlan_proto == ETH_P_8021Q)
 110		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag);
 111	else if (vlan_proto == ETH_P_8021AD)
 112		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag);
 113	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid);
 114
 115	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
 116		 mlx5_eswitch_get_vport_metadata_mask());
 117
 118	MLX5_SET(create_flow_group_in, in, start_flow_index, from);
 119	MLX5_SET(create_flow_group_in, in, end_flow_index, to);
 120
 121	fg = mlx5_create_flow_group(ingress_ft, in);
 122	kvfree(in);
 123	if (IS_ERR(fg))
 124		esw_warn(esw->dev,
 125			 "Failed to create VLAN(proto=%x) flow group for bridge ingress table (err=%pe)\n",
 126			 vlan_proto, fg);
 127
 128	return fg;
 129}
 130
 131static struct mlx5_flow_group *
 132mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw,
 133				       struct mlx5_flow_table *ingress_ft)
 134{
 135	unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM;
 136	unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO;
 137
 138	return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, ingress_ft);
 139}
 140
 141static struct mlx5_flow_group *
 142mlx5_esw_bridge_ingress_qinq_fg_create(struct mlx5_eswitch *esw,
 143				       struct mlx5_flow_table *ingress_ft)
 144{
 145	unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM;
 146	unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO;
 147
 148	return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw,
 149							    ingress_ft);
 150}
 151
 152static struct mlx5_flow_group *
 153mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(unsigned int from, unsigned int to,
 154						    u16 vlan_proto, struct mlx5_eswitch *esw,
 155						    struct mlx5_flow_table *ingress_ft)
 156{
 157	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 158	struct mlx5_flow_group *fg;
 159	u32 *in, *match;
 160
 161	in = kvzalloc(inlen, GFP_KERNEL);
 162	if (!in)
 163		return ERR_PTR(-ENOMEM);
 164
 165	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
 166		 MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2);
 167	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 168
 169	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16);
 170	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0);
 171	if (vlan_proto == ETH_P_8021Q)
 172		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag);
 173	else if (vlan_proto == ETH_P_8021AD)
 174		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag);
 175	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
 176		 mlx5_eswitch_get_vport_metadata_mask());
 177
 178	MLX5_SET(create_flow_group_in, in, start_flow_index, from);
 179	MLX5_SET(create_flow_group_in, in, end_flow_index, to);
 180
 181	fg = mlx5_create_flow_group(ingress_ft, in);
 182	if (IS_ERR(fg))
 183		esw_warn(esw->dev,
 184			 "Failed to create bridge ingress table VLAN filter flow group (err=%pe)\n",
 185			 fg);
 186	kvfree(in);
 187	return fg;
 188}
 189
 190static struct mlx5_flow_group *
 191mlx5_esw_bridge_ingress_vlan_filter_fg_create(struct mlx5_eswitch *esw,
 192					      struct mlx5_flow_table *ingress_ft)
 193{
 194	unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM;
 195	unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO;
 196
 197	return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021Q, esw,
 198								   ingress_ft);
 199}
 200
 201static struct mlx5_flow_group *
 202mlx5_esw_bridge_ingress_qinq_filter_fg_create(struct mlx5_eswitch *esw,
 203					      struct mlx5_flow_table *ingress_ft)
 204{
 205	unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM;
 206	unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO;
 207
 208	return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021AD, esw,
 209								   ingress_ft);
 210}
 211
 212static struct mlx5_flow_group *
 213mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft)
 214{
 215	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 216	struct mlx5_flow_group *fg;
 217	u32 *in, *match;
 218
 219	in = kvzalloc(inlen, GFP_KERNEL);
 220	if (!in)
 221		return ERR_PTR(-ENOMEM);
 222
 223	MLX5_SET(create_flow_group_in, in, match_criteria_enable,
 224		 MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2);
 225	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 226
 227	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16);
 228	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0);
 229
 230	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
 231		 mlx5_eswitch_get_vport_metadata_mask());
 232
 233	MLX5_SET(create_flow_group_in, in, start_flow_index,
 234		 MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM);
 235	MLX5_SET(create_flow_group_in, in, end_flow_index,
 236		 MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO);
 237
 238	fg = mlx5_create_flow_group(ingress_ft, in);
 239	if (IS_ERR(fg))
 240		esw_warn(esw->dev,
 241			 "Failed to create MAC flow group for bridge ingress table (err=%pe)\n",
 242			 fg);
 243
 244	kvfree(in);
 245	return fg;
 246}
 247
 248static struct mlx5_flow_group *
 249mlx5_esw_bridge_egress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto,
 250					    struct mlx5_eswitch *esw,
 251					    struct mlx5_flow_table *egress_ft)
 252{
 253	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 254	struct mlx5_flow_group *fg;
 255	u32 *in, *match;
 256
 257	in = kvzalloc(inlen, GFP_KERNEL);
 258	if (!in)
 259		return ERR_PTR(-ENOMEM);
 260
 261	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
 262	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 263
 264	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16);
 265	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0);
 266	if (vlan_proto == ETH_P_8021Q)
 267		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag);
 268	else if (vlan_proto == ETH_P_8021AD)
 269		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag);
 270	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid);
 271
 272	MLX5_SET(create_flow_group_in, in, start_flow_index, from);
 273	MLX5_SET(create_flow_group_in, in, end_flow_index, to);
 274
 275	fg = mlx5_create_flow_group(egress_ft, in);
 276	if (IS_ERR(fg))
 277		esw_warn(esw->dev,
 278			 "Failed to create VLAN flow group for bridge egress table (err=%pe)\n",
 279			 fg);
 280	kvfree(in);
 281	return fg;
 282}
 283
 284static struct mlx5_flow_group *
 285mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft)
 286{
 287	unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM;
 288	unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO;
 289
 290	return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, egress_ft);
 291}
 292
 293static struct mlx5_flow_group *
 294mlx5_esw_bridge_egress_qinq_fg_create(struct mlx5_eswitch *esw,
 295				      struct mlx5_flow_table *egress_ft)
 296{
 297	unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM;
 298	unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO;
 299
 300	return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, egress_ft);
 301}
 302
 303static struct mlx5_flow_group *
 304mlx5_esw_bridge_egress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft)
 305{
 306	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 307	struct mlx5_flow_group *fg;
 308	u32 *in, *match;
 309
 310	in = kvzalloc(inlen, GFP_KERNEL);
 311	if (!in)
 312		return ERR_PTR(-ENOMEM);
 313
 314	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
 315	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 316
 317	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16);
 318	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0);
 319
 320	MLX5_SET(create_flow_group_in, in, start_flow_index,
 321		 MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM);
 322	MLX5_SET(create_flow_group_in, in, end_flow_index,
 323		 MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO);
 324
 325	fg = mlx5_create_flow_group(egress_ft, in);
 326	if (IS_ERR(fg))
 327		esw_warn(esw->dev,
 328			 "Failed to create bridge egress table MAC flow group (err=%pe)\n",
 329			 fg);
 330	kvfree(in);
 331	return fg;
 332}
 333
 334static struct mlx5_flow_group *
 335mlx5_esw_bridge_egress_miss_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft)
 336{
 337	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 338	struct mlx5_flow_group *fg;
 339	u32 *in, *match;
 340
 341	in = kvzalloc(inlen, GFP_KERNEL);
 342	if (!in)
 343		return ERR_PTR(-ENOMEM);
 344
 345	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2);
 346	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 347
 348	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
 349
 350	MLX5_SET(create_flow_group_in, in, start_flow_index,
 351		 MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM);
 352	MLX5_SET(create_flow_group_in, in, end_flow_index,
 353		 MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO);
 354
 355	fg = mlx5_create_flow_group(egress_ft, in);
 356	if (IS_ERR(fg))
 357		esw_warn(esw->dev,
 358			 "Failed to create bridge egress table miss flow group (err=%pe)\n",
 359			 fg);
 360	kvfree(in);
 361	return fg;
 362}
 363
 364static int
 365mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads)
 366{
 367	struct mlx5_flow_group *mac_fg, *qinq_filter_fg, *qinq_fg, *vlan_filter_fg, *vlan_fg;
 368	struct mlx5_flow_table *ingress_ft, *skip_ft;
 369	struct mlx5_eswitch *esw = br_offloads->esw;
 370	int err;
 371
 372	if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
 373		return -EOPNOTSUPP;
 374
 375	ingress_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE,
 376						  MLX5_ESW_BRIDGE_LEVEL_INGRESS_TABLE,
 377						  esw);
 378	if (IS_ERR(ingress_ft))
 379		return PTR_ERR(ingress_ft);
 380
 381	skip_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE,
 382					       MLX5_ESW_BRIDGE_LEVEL_SKIP_TABLE,
 383					       esw);
 384	if (IS_ERR(skip_ft)) {
 385		err = PTR_ERR(skip_ft);
 386		goto err_skip_tbl;
 387	}
 388
 389	vlan_fg = mlx5_esw_bridge_ingress_vlan_fg_create(esw, ingress_ft);
 390	if (IS_ERR(vlan_fg)) {
 391		err = PTR_ERR(vlan_fg);
 392		goto err_vlan_fg;
 393	}
 394
 395	vlan_filter_fg = mlx5_esw_bridge_ingress_vlan_filter_fg_create(esw, ingress_ft);
 396	if (IS_ERR(vlan_filter_fg)) {
 397		err = PTR_ERR(vlan_filter_fg);
 398		goto err_vlan_filter_fg;
 399	}
 400
 401	qinq_fg = mlx5_esw_bridge_ingress_qinq_fg_create(esw, ingress_ft);
 402	if (IS_ERR(qinq_fg)) {
 403		err = PTR_ERR(qinq_fg);
 404		goto err_qinq_fg;
 405	}
 406
 407	qinq_filter_fg = mlx5_esw_bridge_ingress_qinq_filter_fg_create(esw, ingress_ft);
 408	if (IS_ERR(qinq_filter_fg)) {
 409		err = PTR_ERR(qinq_filter_fg);
 410		goto err_qinq_filter_fg;
 411	}
 412
 413	mac_fg = mlx5_esw_bridge_ingress_mac_fg_create(esw, ingress_ft);
 414	if (IS_ERR(mac_fg)) {
 415		err = PTR_ERR(mac_fg);
 416		goto err_mac_fg;
 417	}
 418
 419	br_offloads->ingress_ft = ingress_ft;
 420	br_offloads->skip_ft = skip_ft;
 421	br_offloads->ingress_vlan_fg = vlan_fg;
 422	br_offloads->ingress_vlan_filter_fg = vlan_filter_fg;
 423	br_offloads->ingress_qinq_fg = qinq_fg;
 424	br_offloads->ingress_qinq_filter_fg = qinq_filter_fg;
 425	br_offloads->ingress_mac_fg = mac_fg;
 426	return 0;
 427
 428err_mac_fg:
 429	mlx5_destroy_flow_group(qinq_filter_fg);
 430err_qinq_filter_fg:
 431	mlx5_destroy_flow_group(qinq_fg);
 432err_qinq_fg:
 433	mlx5_destroy_flow_group(vlan_filter_fg);
 434err_vlan_filter_fg:
 435	mlx5_destroy_flow_group(vlan_fg);
 436err_vlan_fg:
 437	mlx5_destroy_flow_table(skip_ft);
 438err_skip_tbl:
 439	mlx5_destroy_flow_table(ingress_ft);
 440	return err;
 441}
 442
 443static void
 444mlx5_esw_bridge_ingress_table_cleanup(struct mlx5_esw_bridge_offloads *br_offloads)
 445{
 446	mlx5_destroy_flow_group(br_offloads->ingress_mac_fg);
 447	br_offloads->ingress_mac_fg = NULL;
 448	mlx5_destroy_flow_group(br_offloads->ingress_qinq_filter_fg);
 449	br_offloads->ingress_qinq_filter_fg = NULL;
 450	mlx5_destroy_flow_group(br_offloads->ingress_qinq_fg);
 451	br_offloads->ingress_qinq_fg = NULL;
 452	mlx5_destroy_flow_group(br_offloads->ingress_vlan_filter_fg);
 453	br_offloads->ingress_vlan_filter_fg = NULL;
 454	mlx5_destroy_flow_group(br_offloads->ingress_vlan_fg);
 455	br_offloads->ingress_vlan_fg = NULL;
 456	mlx5_destroy_flow_table(br_offloads->skip_ft);
 457	br_offloads->skip_ft = NULL;
 458	mlx5_destroy_flow_table(br_offloads->ingress_ft);
 459	br_offloads->ingress_ft = NULL;
 460}
 461
 462static struct mlx5_flow_handle *
 463mlx5_esw_bridge_egress_miss_flow_create(struct mlx5_flow_table *egress_ft,
 464					struct mlx5_flow_table *skip_ft,
 465					struct mlx5_pkt_reformat *pkt_reformat);
 466
 467static int
 468mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads,
 469				  struct mlx5_esw_bridge *bridge)
 470{
 471	struct mlx5_flow_group *miss_fg = NULL, *mac_fg, *vlan_fg, *qinq_fg;
 472	struct mlx5_pkt_reformat *miss_pkt_reformat = NULL;
 473	struct mlx5_flow_handle *miss_handle = NULL;
 474	struct mlx5_eswitch *esw = br_offloads->esw;
 475	struct mlx5_flow_table *egress_ft;
 476	int err;
 477
 478	egress_ft = mlx5_esw_bridge_table_create(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE,
 479						 MLX5_ESW_BRIDGE_LEVEL_EGRESS_TABLE,
 480						 esw);
 481	if (IS_ERR(egress_ft))
 482		return PTR_ERR(egress_ft);
 483
 484	vlan_fg = mlx5_esw_bridge_egress_vlan_fg_create(esw, egress_ft);
 485	if (IS_ERR(vlan_fg)) {
 486		err = PTR_ERR(vlan_fg);
 487		goto err_vlan_fg;
 488	}
 489
 490	qinq_fg = mlx5_esw_bridge_egress_qinq_fg_create(esw, egress_ft);
 491	if (IS_ERR(qinq_fg)) {
 492		err = PTR_ERR(qinq_fg);
 493		goto err_qinq_fg;
 494	}
 495
 496	mac_fg = mlx5_esw_bridge_egress_mac_fg_create(esw, egress_ft);
 497	if (IS_ERR(mac_fg)) {
 498		err = PTR_ERR(mac_fg);
 499		goto err_mac_fg;
 500	}
 501
 502	if (mlx5_esw_bridge_pkt_reformat_vlan_pop_supported(esw)) {
 503		miss_fg = mlx5_esw_bridge_egress_miss_fg_create(esw, egress_ft);
 504		if (IS_ERR(miss_fg)) {
 505			esw_warn(esw->dev, "Failed to create miss flow group (err=%pe)\n",
 506				 miss_fg);
 507			miss_fg = NULL;
 508			goto skip_miss_flow;
 509		}
 510
 511		miss_pkt_reformat = mlx5_esw_bridge_pkt_reformat_vlan_pop_create(esw);
 512		if (IS_ERR(miss_pkt_reformat)) {
 513			esw_warn(esw->dev,
 514				 "Failed to alloc packet reformat REMOVE_HEADER (err=%pe)\n",
 515				 miss_pkt_reformat);
 516			miss_pkt_reformat = NULL;
 517			mlx5_destroy_flow_group(miss_fg);
 518			miss_fg = NULL;
 519			goto skip_miss_flow;
 520		}
 521
 522		miss_handle = mlx5_esw_bridge_egress_miss_flow_create(egress_ft,
 523								      br_offloads->skip_ft,
 524								      miss_pkt_reformat);
 525		if (IS_ERR(miss_handle)) {
 526			esw_warn(esw->dev, "Failed to create miss flow (err=%pe)\n",
 527				 miss_handle);
 528			miss_handle = NULL;
 529			mlx5_packet_reformat_dealloc(esw->dev, miss_pkt_reformat);
 530			miss_pkt_reformat = NULL;
 531			mlx5_destroy_flow_group(miss_fg);
 532			miss_fg = NULL;
 533			goto skip_miss_flow;
 534		}
 535	}
 536skip_miss_flow:
 537
 538	bridge->egress_ft = egress_ft;
 539	bridge->egress_vlan_fg = vlan_fg;
 540	bridge->egress_qinq_fg = qinq_fg;
 541	bridge->egress_mac_fg = mac_fg;
 542	bridge->egress_miss_fg = miss_fg;
 543	bridge->egress_miss_pkt_reformat = miss_pkt_reformat;
 544	bridge->egress_miss_handle = miss_handle;
 545	return 0;
 546
 547err_mac_fg:
 548	mlx5_destroy_flow_group(qinq_fg);
 549err_qinq_fg:
 550	mlx5_destroy_flow_group(vlan_fg);
 551err_vlan_fg:
 552	mlx5_destroy_flow_table(egress_ft);
 553	return err;
 554}
 555
 556static void
 557mlx5_esw_bridge_egress_table_cleanup(struct mlx5_esw_bridge *bridge)
 558{
 559	if (bridge->egress_miss_handle)
 560		mlx5_del_flow_rules(bridge->egress_miss_handle);
 561	if (bridge->egress_miss_pkt_reformat)
 562		mlx5_packet_reformat_dealloc(bridge->br_offloads->esw->dev,
 563					     bridge->egress_miss_pkt_reformat);
 564	if (bridge->egress_miss_fg)
 565		mlx5_destroy_flow_group(bridge->egress_miss_fg);
 566	mlx5_destroy_flow_group(bridge->egress_mac_fg);
 567	mlx5_destroy_flow_group(bridge->egress_qinq_fg);
 568	mlx5_destroy_flow_group(bridge->egress_vlan_fg);
 569	mlx5_destroy_flow_table(bridge->egress_ft);
 570}
 571
 572static struct mlx5_flow_handle *
 573mlx5_esw_bridge_ingress_flow_with_esw_create(u16 vport_num, const unsigned char *addr,
 574					     struct mlx5_esw_bridge_vlan *vlan,
 575					     struct mlx5_fc *counter,
 576					     struct mlx5_esw_bridge *bridge,
 577					     struct mlx5_eswitch *esw)
 578{
 579	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
 580	struct mlx5_flow_act flow_act = {
 581		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_COUNT,
 582		.flags = FLOW_ACT_NO_APPEND,
 583	};
 584	struct mlx5_flow_destination dests[2] = {};
 585	struct mlx5_flow_spec *rule_spec;
 586	struct mlx5_flow_handle *handle;
 587	u8 *smac_v, *smac_c;
 588
 589	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
 590	if (!rule_spec)
 591		return ERR_PTR(-ENOMEM);
 592
 593	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2;
 594
 595	smac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value,
 596			      outer_headers.smac_47_16);
 597	ether_addr_copy(smac_v, addr);
 598	smac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria,
 599			      outer_headers.smac_47_16);
 600	eth_broadcast_addr(smac_c);
 601
 602	MLX5_SET(fte_match_param, rule_spec->match_criteria,
 603		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
 604	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0,
 605		 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
 606
 607	if (vlan && vlan->pkt_reformat_push) {
 608		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
 609			MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
 610		flow_act.pkt_reformat = vlan->pkt_reformat_push;
 611		flow_act.modify_hdr = vlan->pkt_mod_hdr_push_mark;
 612	} else if (vlan) {
 613		if (bridge->vlan_proto == ETH_P_8021Q) {
 614			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 615					 outer_headers.cvlan_tag);
 616			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 617					 outer_headers.cvlan_tag);
 618		} else if (bridge->vlan_proto == ETH_P_8021AD) {
 619			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 620					 outer_headers.svlan_tag);
 621			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 622					 outer_headers.svlan_tag);
 623		}
 624		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 625				 outer_headers.first_vid);
 626		MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid,
 627			 vlan->vid);
 628	}
 629
 630	dests[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 631	dests[0].ft = bridge->egress_ft;
 632	dests[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
 633	dests[1].counter = counter;
 634
 635	handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, dests,
 636				     ARRAY_SIZE(dests));
 637
 638	kvfree(rule_spec);
 639	return handle;
 640}
 641
 642static struct mlx5_flow_handle *
 643mlx5_esw_bridge_ingress_flow_create(u16 vport_num, const unsigned char *addr,
 644				    struct mlx5_esw_bridge_vlan *vlan,
 645				    struct mlx5_fc *counter,
 646				    struct mlx5_esw_bridge *bridge)
 647{
 648	return mlx5_esw_bridge_ingress_flow_with_esw_create(vport_num, addr, vlan, counter,
 649							    bridge, bridge->br_offloads->esw);
 650}
 651
 652static struct mlx5_flow_handle *
 653mlx5_esw_bridge_ingress_flow_peer_create(u16 vport_num, u16 esw_owner_vhca_id,
 654					 const unsigned char *addr,
 655					 struct mlx5_esw_bridge_vlan *vlan,
 656					 struct mlx5_fc *counter,
 657					 struct mlx5_esw_bridge *bridge)
 658{
 659	struct mlx5_devcom_comp_dev *devcom = bridge->br_offloads->esw->devcom, *pos;
 660	struct mlx5_eswitch *tmp, *peer_esw = NULL;
 661	static struct mlx5_flow_handle *handle;
 662
 663	if (!mlx5_devcom_for_each_peer_begin(devcom))
 664		return ERR_PTR(-ENODEV);
 665
 666	mlx5_devcom_for_each_peer_entry(devcom, tmp, pos) {
 667		if (mlx5_esw_is_owner(tmp, vport_num, esw_owner_vhca_id)) {
 668			peer_esw = tmp;
 669			break;
 670		}
 671	}
 672
 673	if (!peer_esw) {
 674		handle = ERR_PTR(-ENODEV);
 675		goto out;
 676	}
 677
 678	handle = mlx5_esw_bridge_ingress_flow_with_esw_create(vport_num, addr, vlan, counter,
 679							      bridge, peer_esw);
 680
 681out:
 682	mlx5_devcom_for_each_peer_end(devcom);
 683	return handle;
 684}
 685
 686static struct mlx5_flow_handle *
 687mlx5_esw_bridge_ingress_filter_flow_create(u16 vport_num, const unsigned char *addr,
 688					   struct mlx5_esw_bridge *bridge)
 689{
 690	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
 691	struct mlx5_flow_destination dest = {
 692		.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
 693		.ft = br_offloads->skip_ft,
 694	};
 695	struct mlx5_flow_act flow_act = {
 696		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 697		.flags = FLOW_ACT_NO_APPEND,
 698	};
 699	struct mlx5_flow_spec *rule_spec;
 700	struct mlx5_flow_handle *handle;
 701	u8 *smac_v, *smac_c;
 702
 703	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
 704	if (!rule_spec)
 705		return ERR_PTR(-ENOMEM);
 706
 707	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2;
 708
 709	smac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value,
 710			      outer_headers.smac_47_16);
 711	ether_addr_copy(smac_v, addr);
 712	smac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria,
 713			      outer_headers.smac_47_16);
 714	eth_broadcast_addr(smac_c);
 715
 716	MLX5_SET(fte_match_param, rule_spec->match_criteria,
 717		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
 718	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0,
 719		 mlx5_eswitch_get_vport_metadata_for_match(br_offloads->esw, vport_num));
 720
 721	if (bridge->vlan_proto == ETH_P_8021Q) {
 722		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 723				 outer_headers.cvlan_tag);
 724		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 725				 outer_headers.cvlan_tag);
 726	} else if (bridge->vlan_proto == ETH_P_8021AD) {
 727		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 728				 outer_headers.svlan_tag);
 729		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 730				 outer_headers.svlan_tag);
 731	}
 732
 733	handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, &dest, 1);
 734
 735	kvfree(rule_spec);
 736	return handle;
 737}
 738
 739static struct mlx5_flow_handle *
 740mlx5_esw_bridge_egress_flow_create(u16 vport_num, u16 esw_owner_vhca_id, const unsigned char *addr,
 741				   struct mlx5_esw_bridge_vlan *vlan,
 742				   struct mlx5_esw_bridge *bridge)
 743{
 744	struct mlx5_flow_destination dest = {
 745		.type = MLX5_FLOW_DESTINATION_TYPE_VPORT,
 746		.vport.num = vport_num,
 747	};
 748	struct mlx5_flow_act flow_act = {
 749		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
 750		.flags = FLOW_ACT_NO_APPEND,
 751	};
 752	struct mlx5_flow_spec *rule_spec;
 753	struct mlx5_flow_handle *handle;
 754	u8 *dmac_v, *dmac_c;
 755
 756	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
 757	if (!rule_spec)
 758		return ERR_PTR(-ENOMEM);
 759
 760	if (MLX5_CAP_ESW_FLOWTABLE(bridge->br_offloads->esw->dev, flow_source) &&
 761	    vport_num == MLX5_VPORT_UPLINK)
 762		rule_spec->flow_context.flow_source =
 763			MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
 764	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 765
 766	dmac_v = MLX5_ADDR_OF(fte_match_param, rule_spec->match_value,
 767			      outer_headers.dmac_47_16);
 768	ether_addr_copy(dmac_v, addr);
 769	dmac_c = MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria,
 770			      outer_headers.dmac_47_16);
 771	eth_broadcast_addr(dmac_c);
 772
 773	if (vlan) {
 774		if (vlan->pkt_reformat_pop) {
 775			flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
 776			flow_act.pkt_reformat = vlan->pkt_reformat_pop;
 777		}
 778
 779		if (bridge->vlan_proto == ETH_P_8021Q) {
 780			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 781					 outer_headers.cvlan_tag);
 782			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 783					 outer_headers.cvlan_tag);
 784		} else if (bridge->vlan_proto == ETH_P_8021AD) {
 785			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 786					 outer_headers.svlan_tag);
 787			MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value,
 788					 outer_headers.svlan_tag);
 789		}
 790		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 791				 outer_headers.first_vid);
 792		MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid,
 793			 vlan->vid);
 794	}
 795
 796	if (MLX5_CAP_ESW(bridge->br_offloads->esw->dev, merged_eswitch)) {
 797		dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID;
 798		dest.vport.vhca_id = esw_owner_vhca_id;
 799	}
 800	handle = mlx5_add_flow_rules(bridge->egress_ft, rule_spec, &flow_act, &dest, 1);
 801
 802	kvfree(rule_spec);
 803	return handle;
 804}
 805
 806static struct mlx5_flow_handle *
 807mlx5_esw_bridge_egress_miss_flow_create(struct mlx5_flow_table *egress_ft,
 808					struct mlx5_flow_table *skip_ft,
 809					struct mlx5_pkt_reformat *pkt_reformat)
 810{
 811	struct mlx5_flow_destination dest = {
 812		.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE,
 813		.ft = skip_ft,
 814	};
 815	struct mlx5_flow_act flow_act = {
 816		.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
 817		MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT,
 818		.flags = FLOW_ACT_NO_APPEND,
 819		.pkt_reformat = pkt_reformat,
 820	};
 821	struct mlx5_flow_spec *rule_spec;
 822	struct mlx5_flow_handle *handle;
 823
 824	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
 825	if (!rule_spec)
 826		return ERR_PTR(-ENOMEM);
 827
 828	rule_spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
 829
 830	MLX5_SET(fte_match_param, rule_spec->match_criteria,
 831		 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
 832	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_1,
 833		 ESW_TUN_BRIDGE_INGRESS_PUSH_VLAN_MARK);
 834
 835	handle = mlx5_add_flow_rules(egress_ft, rule_spec, &flow_act, &dest, 1);
 836
 837	kvfree(rule_spec);
 838	return handle;
 839}
 840
 841static struct mlx5_esw_bridge *mlx5_esw_bridge_create(struct net_device *br_netdev,
 842						      struct mlx5_esw_bridge_offloads *br_offloads)
 843{
 844	struct mlx5_esw_bridge *bridge;
 845	int err;
 846
 847	bridge = kvzalloc(sizeof(*bridge), GFP_KERNEL);
 848	if (!bridge)
 849		return ERR_PTR(-ENOMEM);
 850
 851	bridge->br_offloads = br_offloads;
 852	err = mlx5_esw_bridge_egress_table_init(br_offloads, bridge);
 853	if (err)
 854		goto err_egress_tbl;
 855
 856	err = rhashtable_init(&bridge->fdb_ht, &fdb_ht_params);
 857	if (err)
 858		goto err_fdb_ht;
 859
 860	err = mlx5_esw_bridge_mdb_init(bridge);
 861	if (err)
 862		goto err_mdb_ht;
 863
 864	INIT_LIST_HEAD(&bridge->fdb_list);
 865	bridge->ifindex = br_netdev->ifindex;
 866	bridge->refcnt = 1;
 867	bridge->ageing_time = clock_t_to_jiffies(BR_DEFAULT_AGEING_TIME);
 868	bridge->vlan_proto = ETH_P_8021Q;
 869	list_add(&bridge->list, &br_offloads->bridges);
 870	mlx5_esw_bridge_debugfs_init(br_netdev, bridge);
 871
 872	return bridge;
 873
 874err_mdb_ht:
 875	rhashtable_destroy(&bridge->fdb_ht);
 876err_fdb_ht:
 877	mlx5_esw_bridge_egress_table_cleanup(bridge);
 878err_egress_tbl:
 879	kvfree(bridge);
 880	return ERR_PTR(err);
 881}
 882
 883static void mlx5_esw_bridge_get(struct mlx5_esw_bridge *bridge)
 884{
 885	bridge->refcnt++;
 886}
 887
 888static void mlx5_esw_bridge_put(struct mlx5_esw_bridge_offloads *br_offloads,
 889				struct mlx5_esw_bridge *bridge)
 890{
 891	if (--bridge->refcnt)
 892		return;
 893
 894	mlx5_esw_bridge_debugfs_cleanup(bridge);
 895	mlx5_esw_bridge_egress_table_cleanup(bridge);
 896	mlx5_esw_bridge_mcast_disable(bridge);
 897	list_del(&bridge->list);
 898	mlx5_esw_bridge_mdb_cleanup(bridge);
 899	rhashtable_destroy(&bridge->fdb_ht);
 900	kvfree(bridge);
 901
 902	if (list_empty(&br_offloads->bridges))
 903		mlx5_esw_bridge_ingress_table_cleanup(br_offloads);
 904}
 905
 906static struct mlx5_esw_bridge *
 907mlx5_esw_bridge_lookup(struct net_device *br_netdev, struct mlx5_esw_bridge_offloads *br_offloads)
 908{
 909	struct mlx5_esw_bridge *bridge;
 910
 911	ASSERT_RTNL();
 912
 913	list_for_each_entry(bridge, &br_offloads->bridges, list) {
 914		if (bridge->ifindex == br_netdev->ifindex) {
 915			mlx5_esw_bridge_get(bridge);
 916			return bridge;
 917		}
 918	}
 919
 920	if (!br_offloads->ingress_ft) {
 921		int err = mlx5_esw_bridge_ingress_table_init(br_offloads);
 922
 923		if (err)
 924			return ERR_PTR(err);
 925	}
 926
 927	bridge = mlx5_esw_bridge_create(br_netdev, br_offloads);
 928	if (IS_ERR(bridge) && list_empty(&br_offloads->bridges))
 929		mlx5_esw_bridge_ingress_table_cleanup(br_offloads);
 930	return bridge;
 931}
 932
 933static unsigned long mlx5_esw_bridge_port_key_from_data(u16 vport_num, u16 esw_owner_vhca_id)
 934{
 935	return vport_num | (unsigned long)esw_owner_vhca_id << sizeof(vport_num) * BITS_PER_BYTE;
 936}
 937
 938unsigned long mlx5_esw_bridge_port_key(struct mlx5_esw_bridge_port *port)
 939{
 940	return mlx5_esw_bridge_port_key_from_data(port->vport_num, port->esw_owner_vhca_id);
 941}
 942
 943static int mlx5_esw_bridge_port_insert(struct mlx5_esw_bridge_port *port,
 944				       struct mlx5_esw_bridge_offloads *br_offloads)
 945{
 946	return xa_insert(&br_offloads->ports, mlx5_esw_bridge_port_key(port), port, GFP_KERNEL);
 947}
 948
 949static struct mlx5_esw_bridge_port *
 950mlx5_esw_bridge_port_lookup(u16 vport_num, u16 esw_owner_vhca_id,
 951			    struct mlx5_esw_bridge_offloads *br_offloads)
 952{
 953	return xa_load(&br_offloads->ports, mlx5_esw_bridge_port_key_from_data(vport_num,
 954									       esw_owner_vhca_id));
 955}
 956
 957static void mlx5_esw_bridge_port_erase(struct mlx5_esw_bridge_port *port,
 958				       struct mlx5_esw_bridge_offloads *br_offloads)
 959{
 960	xa_erase(&br_offloads->ports, mlx5_esw_bridge_port_key(port));
 961}
 962
 963static struct mlx5_esw_bridge *
 964mlx5_esw_bridge_from_port_lookup(u16 vport_num, u16 esw_owner_vhca_id,
 965				 struct mlx5_esw_bridge_offloads *br_offloads)
 966{
 967	struct mlx5_esw_bridge_port *port;
 968
 969	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
 970	if (!port)
 971		return NULL;
 972
 973	return port->bridge;
 974}
 975
 976static void mlx5_esw_bridge_fdb_entry_refresh(struct mlx5_esw_bridge_fdb_entry *entry)
 977{
 978	trace_mlx5_esw_bridge_fdb_entry_refresh(entry);
 979
 980	mlx5_esw_bridge_fdb_offload_notify(entry->dev, entry->key.addr,
 981					   entry->key.vid,
 982					   SWITCHDEV_FDB_ADD_TO_BRIDGE);
 983}
 984
 985static void
 986mlx5_esw_bridge_fdb_entry_cleanup(struct mlx5_esw_bridge_fdb_entry *entry,
 987				  struct mlx5_esw_bridge *bridge)
 988{
 989	trace_mlx5_esw_bridge_fdb_entry_cleanup(entry);
 990
 991	rhashtable_remove_fast(&bridge->fdb_ht, &entry->ht_node, fdb_ht_params);
 992	mlx5_del_flow_rules(entry->egress_handle);
 993	if (entry->filter_handle)
 994		mlx5_del_flow_rules(entry->filter_handle);
 995	mlx5_del_flow_rules(entry->ingress_handle);
 996	mlx5_fc_destroy(bridge->br_offloads->esw->dev, entry->ingress_counter);
 997	list_del(&entry->vlan_list);
 998	list_del(&entry->list);
 999	kvfree(entry);
1000}
1001
1002static void
1003mlx5_esw_bridge_fdb_entry_notify_and_cleanup(struct mlx5_esw_bridge_fdb_entry *entry,
1004					     struct mlx5_esw_bridge *bridge)
1005{
1006	mlx5_esw_bridge_fdb_del_notify(entry);
1007	mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge);
1008}
1009
1010static void mlx5_esw_bridge_fdb_flush(struct mlx5_esw_bridge *bridge)
1011{
1012	struct mlx5_esw_bridge_fdb_entry *entry, *tmp;
1013
1014	list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list)
1015		mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
1016}
1017
1018static struct mlx5_esw_bridge_vlan *
1019mlx5_esw_bridge_vlan_lookup(u16 vid, struct mlx5_esw_bridge_port *port)
1020{
1021	return xa_load(&port->vlans, vid);
1022}
1023
1024static int
1025mlx5_esw_bridge_vlan_push_create(u16 vlan_proto, struct mlx5_esw_bridge_vlan *vlan,
1026				 struct mlx5_eswitch *esw)
1027{
1028	struct {
1029		__be16	h_vlan_proto;
1030		__be16	h_vlan_TCI;
1031	} vlan_hdr = { htons(vlan_proto), htons(vlan->vid) };
1032	struct mlx5_pkt_reformat_params reformat_params = {};
1033	struct mlx5_pkt_reformat *pkt_reformat;
1034
1035	if (!BIT(MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat_insert)) ||
1036	    MLX5_CAP_GEN_2(esw->dev, max_reformat_insert_size) < sizeof(vlan_hdr) ||
1037	    MLX5_CAP_GEN_2(esw->dev, max_reformat_insert_offset) <
1038	    offsetof(struct vlan_ethhdr, h_vlan_proto)) {
1039		esw_warn(esw->dev, "Packet reformat INSERT_HEADER is not supported\n");
1040		return -EOPNOTSUPP;
1041	}
1042
1043	reformat_params.type = MLX5_REFORMAT_TYPE_INSERT_HDR;
1044	reformat_params.param_0 = MLX5_REFORMAT_CONTEXT_ANCHOR_MAC_START;
1045	reformat_params.param_1 = offsetof(struct vlan_ethhdr, h_vlan_proto);
1046	reformat_params.size = sizeof(vlan_hdr);
1047	reformat_params.data = &vlan_hdr;
1048	pkt_reformat = mlx5_packet_reformat_alloc(esw->dev,
1049						  &reformat_params,
1050						  MLX5_FLOW_NAMESPACE_FDB);
1051	if (IS_ERR(pkt_reformat)) {
1052		esw_warn(esw->dev, "Failed to alloc packet reformat INSERT_HEADER (err=%pe)\n",
1053			 pkt_reformat);
1054		return PTR_ERR(pkt_reformat);
1055	}
1056
1057	vlan->pkt_reformat_push = pkt_reformat;
1058	return 0;
1059}
1060
1061static void
1062mlx5_esw_bridge_vlan_push_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1063{
1064	mlx5_packet_reformat_dealloc(esw->dev, vlan->pkt_reformat_push);
1065	vlan->pkt_reformat_push = NULL;
1066}
1067
1068static int
1069mlx5_esw_bridge_vlan_pop_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1070{
1071	struct mlx5_pkt_reformat *pkt_reformat;
1072
1073	if (!mlx5_esw_bridge_pkt_reformat_vlan_pop_supported(esw)) {
1074		esw_warn(esw->dev, "Packet reformat REMOVE_HEADER is not supported\n");
1075		return -EOPNOTSUPP;
1076	}
1077
1078	pkt_reformat = mlx5_esw_bridge_pkt_reformat_vlan_pop_create(esw);
1079	if (IS_ERR(pkt_reformat)) {
1080		esw_warn(esw->dev, "Failed to alloc packet reformat REMOVE_HEADER (err=%pe)\n",
1081			 pkt_reformat);
1082		return PTR_ERR(pkt_reformat);
1083	}
1084
1085	vlan->pkt_reformat_pop = pkt_reformat;
1086	return 0;
1087}
1088
1089static void
1090mlx5_esw_bridge_vlan_pop_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1091{
1092	mlx5_packet_reformat_dealloc(esw->dev, vlan->pkt_reformat_pop);
1093	vlan->pkt_reformat_pop = NULL;
1094}
1095
1096static int
1097mlx5_esw_bridge_vlan_push_mark_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1098{
1099	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
1100	struct mlx5_modify_hdr *pkt_mod_hdr;
1101
1102	MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
1103	MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
1104	MLX5_SET(set_action_in, action, offset, 8);
1105	MLX5_SET(set_action_in, action, length, ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS);
1106	MLX5_SET(set_action_in, action, data, ESW_TUN_BRIDGE_INGRESS_PUSH_VLAN);
1107
1108	pkt_mod_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB, 1, action);
1109	if (IS_ERR(pkt_mod_hdr))
1110		return PTR_ERR(pkt_mod_hdr);
1111
1112	vlan->pkt_mod_hdr_push_mark = pkt_mod_hdr;
1113	return 0;
1114}
1115
1116static void
1117mlx5_esw_bridge_vlan_push_mark_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1118{
1119	mlx5_modify_header_dealloc(esw->dev, vlan->pkt_mod_hdr_push_mark);
1120	vlan->pkt_mod_hdr_push_mark = NULL;
1121}
1122
1123static int
1124mlx5_esw_bridge_vlan_push_pop_fhs_create(u16 vlan_proto, struct mlx5_esw_bridge_port *port,
1125					 struct mlx5_esw_bridge_vlan *vlan)
1126{
1127	return mlx5_esw_bridge_vlan_mcast_init(vlan_proto, port, vlan);
1128}
1129
1130static void
1131mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(struct mlx5_esw_bridge_vlan *vlan)
1132{
1133	mlx5_esw_bridge_vlan_mcast_cleanup(vlan);
1134}
1135
1136static int
1137mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_bridge_port *port,
1138				     struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw)
1139{
1140	int err;
1141
1142	if (flags & BRIDGE_VLAN_INFO_PVID) {
1143		err = mlx5_esw_bridge_vlan_push_create(vlan_proto, vlan, esw);
1144		if (err)
1145			return err;
1146
1147		err = mlx5_esw_bridge_vlan_push_mark_create(vlan, esw);
1148		if (err)
1149			goto err_vlan_push_mark;
1150	}
1151
1152	if (flags & BRIDGE_VLAN_INFO_UNTAGGED) {
1153		err = mlx5_esw_bridge_vlan_pop_create(vlan, esw);
1154		if (err)
1155			goto err_vlan_pop;
1156
1157		err = mlx5_esw_bridge_vlan_push_pop_fhs_create(vlan_proto, port, vlan);
1158		if (err)
1159			goto err_vlan_pop_fhs;
1160	}
1161
1162	return 0;
1163
1164err_vlan_pop_fhs:
1165	mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
1166err_vlan_pop:
1167	if (vlan->pkt_mod_hdr_push_mark)
1168		mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw);
1169err_vlan_push_mark:
1170	if (vlan->pkt_reformat_push)
1171		mlx5_esw_bridge_vlan_push_cleanup(vlan, esw);
1172	return err;
1173}
1174
1175static struct mlx5_esw_bridge_vlan *
1176mlx5_esw_bridge_vlan_create(u16 vlan_proto, u16 vid, u16 flags, struct mlx5_esw_bridge_port *port,
1177			    struct mlx5_eswitch *esw)
1178{
1179	struct mlx5_esw_bridge_vlan *vlan;
1180	int err;
1181
1182	vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL);
1183	if (!vlan)
1184		return ERR_PTR(-ENOMEM);
1185
1186	vlan->vid = vid;
1187	vlan->flags = flags;
1188	INIT_LIST_HEAD(&vlan->fdb_list);
1189
1190	err = mlx5_esw_bridge_vlan_push_pop_create(vlan_proto, flags, port, vlan, esw);
1191	if (err)
1192		goto err_vlan_push_pop;
1193
1194	err = xa_insert(&port->vlans, vid, vlan, GFP_KERNEL);
1195	if (err)
1196		goto err_xa_insert;
1197
1198	trace_mlx5_esw_bridge_vlan_create(vlan);
1199	return vlan;
1200
1201err_xa_insert:
1202	if (vlan->mcast_handle)
1203		mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(vlan);
1204	if (vlan->pkt_reformat_pop)
1205		mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
1206	if (vlan->pkt_mod_hdr_push_mark)
1207		mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw);
1208	if (vlan->pkt_reformat_push)
1209		mlx5_esw_bridge_vlan_push_cleanup(vlan, esw);
1210err_vlan_push_pop:
1211	kvfree(vlan);
1212	return ERR_PTR(err);
1213}
1214
1215static void mlx5_esw_bridge_vlan_erase(struct mlx5_esw_bridge_port *port,
1216				       struct mlx5_esw_bridge_vlan *vlan)
1217{
1218	xa_erase(&port->vlans, vlan->vid);
1219}
1220
1221static void mlx5_esw_bridge_vlan_flush(struct mlx5_esw_bridge_port *port,
1222				       struct mlx5_esw_bridge_vlan *vlan,
1223				       struct mlx5_esw_bridge *bridge)
1224{
1225	struct mlx5_eswitch *esw = bridge->br_offloads->esw;
1226	struct mlx5_esw_bridge_fdb_entry *entry, *tmp;
1227
1228	list_for_each_entry_safe(entry, tmp, &vlan->fdb_list, vlan_list)
1229		mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
1230	mlx5_esw_bridge_port_mdb_vlan_flush(port, vlan);
1231
1232	if (vlan->mcast_handle)
1233		mlx5_esw_bridge_vlan_push_pop_fhs_cleanup(vlan);
1234	if (vlan->pkt_reformat_pop)
1235		mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw);
1236	if (vlan->pkt_mod_hdr_push_mark)
1237		mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw);
1238	if (vlan->pkt_reformat_push)
1239		mlx5_esw_bridge_vlan_push_cleanup(vlan, esw);
1240}
1241
1242static void mlx5_esw_bridge_vlan_cleanup(struct mlx5_esw_bridge_port *port,
1243					 struct mlx5_esw_bridge_vlan *vlan,
1244					 struct mlx5_esw_bridge *bridge)
1245{
1246	trace_mlx5_esw_bridge_vlan_cleanup(vlan);
1247	mlx5_esw_bridge_vlan_flush(port, vlan, bridge);
1248	mlx5_esw_bridge_vlan_erase(port, vlan);
1249	kvfree(vlan);
1250}
1251
1252static void mlx5_esw_bridge_port_vlans_flush(struct mlx5_esw_bridge_port *port,
1253					     struct mlx5_esw_bridge *bridge)
1254{
1255	struct mlx5_esw_bridge_vlan *vlan;
1256	unsigned long index;
1257
1258	xa_for_each(&port->vlans, index, vlan)
1259		mlx5_esw_bridge_vlan_cleanup(port, vlan, bridge);
1260}
1261
1262static int mlx5_esw_bridge_port_vlans_recreate(struct mlx5_esw_bridge_port *port,
1263					       struct mlx5_esw_bridge *bridge)
1264{
1265	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
1266	struct mlx5_esw_bridge_vlan *vlan;
1267	unsigned long i;
1268	int err;
1269
1270	xa_for_each(&port->vlans, i, vlan) {
1271		mlx5_esw_bridge_vlan_flush(port, vlan, bridge);
1272		err = mlx5_esw_bridge_vlan_push_pop_create(bridge->vlan_proto, vlan->flags, port,
1273							   vlan, br_offloads->esw);
1274		if (err) {
1275			esw_warn(br_offloads->esw->dev,
1276				 "Failed to create VLAN=%u(proto=%x) push/pop actions (vport=%u,err=%d)\n",
1277				 vlan->vid, bridge->vlan_proto, port->vport_num,
1278				 err);
1279			return err;
1280		}
1281	}
1282
1283	return 0;
1284}
1285
1286static int
1287mlx5_esw_bridge_vlans_recreate(struct mlx5_esw_bridge *bridge)
1288{
1289	struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads;
1290	struct mlx5_esw_bridge_port *port;
1291	unsigned long i;
1292	int err;
1293
1294	xa_for_each(&br_offloads->ports, i, port) {
1295		if (port->bridge != bridge)
1296			continue;
1297
1298		err = mlx5_esw_bridge_port_vlans_recreate(port, bridge);
1299		if (err)
1300			return err;
1301	}
1302
1303	return 0;
1304}
1305
1306static struct mlx5_esw_bridge_vlan *
1307mlx5_esw_bridge_port_vlan_lookup(u16 vid, u16 vport_num, u16 esw_owner_vhca_id,
1308				 struct mlx5_esw_bridge *bridge, struct mlx5_eswitch *esw)
1309{
1310	struct mlx5_esw_bridge_port *port;
1311	struct mlx5_esw_bridge_vlan *vlan;
1312
1313	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, bridge->br_offloads);
1314	if (!port) {
1315		/* FDB is added asynchronously on wq while port might have been deleted
1316		 * concurrently. Report on 'info' logging level and skip the FDB offload.
1317		 */
1318		esw_info(esw->dev, "Failed to lookup bridge port (vport=%u)\n", vport_num);
1319		return ERR_PTR(-EINVAL);
1320	}
1321
1322	vlan = mlx5_esw_bridge_vlan_lookup(vid, port);
1323	if (!vlan) {
1324		/* FDB is added asynchronously on wq while vlan might have been deleted
1325		 * concurrently. Report on 'info' logging level and skip the FDB offload.
1326		 */
1327		esw_info(esw->dev, "Failed to lookup bridge port vlan metadata (vport=%u)\n",
1328			 vport_num);
1329		return ERR_PTR(-EINVAL);
1330	}
1331
1332	return vlan;
1333}
1334
1335static struct mlx5_esw_bridge_fdb_entry *
1336mlx5_esw_bridge_fdb_lookup(struct mlx5_esw_bridge *bridge,
1337			   const unsigned char *addr, u16 vid)
1338{
1339	struct mlx5_esw_bridge_fdb_key key = {};
1340
1341	ether_addr_copy(key.addr, addr);
1342	key.vid = vid;
1343	return rhashtable_lookup_fast(&bridge->fdb_ht, &key, fdb_ht_params);
1344}
1345
1346static struct mlx5_esw_bridge_fdb_entry *
1347mlx5_esw_bridge_fdb_entry_init(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1348			       const unsigned char *addr, u16 vid, bool added_by_user, bool peer,
1349			       struct mlx5_eswitch *esw, struct mlx5_esw_bridge *bridge)
1350{
1351	struct mlx5_esw_bridge_vlan *vlan = NULL;
1352	struct mlx5_esw_bridge_fdb_entry *entry;
1353	struct mlx5_flow_handle *handle;
1354	struct mlx5_fc *counter;
1355	int err;
1356
1357	if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG && vid) {
1358		vlan = mlx5_esw_bridge_port_vlan_lookup(vid, vport_num, esw_owner_vhca_id, bridge,
1359							esw);
1360		if (IS_ERR(vlan))
1361			return ERR_CAST(vlan);
1362	}
1363
1364	entry = mlx5_esw_bridge_fdb_lookup(bridge, addr, vid);
1365	if (entry)
1366		mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
1367
1368	entry = kvzalloc(sizeof(*entry), GFP_KERNEL);
1369	if (!entry)
1370		return ERR_PTR(-ENOMEM);
1371
1372	ether_addr_copy(entry->key.addr, addr);
1373	entry->key.vid = vid;
1374	entry->dev = dev;
1375	entry->vport_num = vport_num;
1376	entry->esw_owner_vhca_id = esw_owner_vhca_id;
1377	entry->lastuse = jiffies;
1378	if (added_by_user)
1379		entry->flags |= MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER;
1380	if (peer)
1381		entry->flags |= MLX5_ESW_BRIDGE_FLAG_PEER;
1382
1383	counter = mlx5_fc_create(esw->dev, true);
1384	if (IS_ERR(counter)) {
1385		err = PTR_ERR(counter);
1386		goto err_ingress_fc_create;
1387	}
1388	entry->ingress_counter = counter;
1389
1390	handle = peer ?
1391		mlx5_esw_bridge_ingress_flow_peer_create(vport_num, esw_owner_vhca_id,
1392							 addr, vlan, counter, bridge) :
1393		mlx5_esw_bridge_ingress_flow_create(vport_num, addr, vlan,
1394						    counter, bridge);
1395	if (IS_ERR(handle)) {
1396		err = PTR_ERR(handle);
1397		esw_warn(esw->dev, "Failed to create ingress flow(vport=%u,err=%d,peer=%d)\n",
1398			 vport_num, err, peer);
1399		goto err_ingress_flow_create;
1400	}
1401	entry->ingress_handle = handle;
1402
1403	if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG) {
1404		handle = mlx5_esw_bridge_ingress_filter_flow_create(vport_num, addr, bridge);
1405		if (IS_ERR(handle)) {
1406			err = PTR_ERR(handle);
1407			esw_warn(esw->dev, "Failed to create ingress filter(vport=%u,err=%d)\n",
1408				 vport_num, err);
1409			goto err_ingress_filter_flow_create;
1410		}
1411		entry->filter_handle = handle;
1412	}
1413
1414	handle = mlx5_esw_bridge_egress_flow_create(vport_num, esw_owner_vhca_id, addr, vlan,
1415						    bridge);
1416	if (IS_ERR(handle)) {
1417		err = PTR_ERR(handle);
1418		esw_warn(esw->dev, "Failed to create egress flow(vport=%u,err=%d)\n",
1419			 vport_num, err);
1420		goto err_egress_flow_create;
1421	}
1422	entry->egress_handle = handle;
1423
1424	err = rhashtable_insert_fast(&bridge->fdb_ht, &entry->ht_node, fdb_ht_params);
1425	if (err) {
1426		esw_warn(esw->dev, "Failed to insert FDB flow(vport=%u,err=%d)\n", vport_num, err);
1427		goto err_ht_init;
1428	}
1429
1430	if (vlan)
1431		list_add(&entry->vlan_list, &vlan->fdb_list);
1432	else
1433		INIT_LIST_HEAD(&entry->vlan_list);
1434	list_add(&entry->list, &bridge->fdb_list);
1435
1436	trace_mlx5_esw_bridge_fdb_entry_init(entry);
1437	return entry;
1438
1439err_ht_init:
1440	mlx5_del_flow_rules(entry->egress_handle);
1441err_egress_flow_create:
1442	if (entry->filter_handle)
1443		mlx5_del_flow_rules(entry->filter_handle);
1444err_ingress_filter_flow_create:
1445	mlx5_del_flow_rules(entry->ingress_handle);
1446err_ingress_flow_create:
1447	mlx5_fc_destroy(esw->dev, entry->ingress_counter);
1448err_ingress_fc_create:
1449	kvfree(entry);
1450	return ERR_PTR(err);
1451}
1452
1453int mlx5_esw_bridge_ageing_time_set(u16 vport_num, u16 esw_owner_vhca_id, unsigned long ageing_time,
1454				    struct mlx5_esw_bridge_offloads *br_offloads)
1455{
1456	struct mlx5_esw_bridge *bridge;
1457
1458	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1459	if (!bridge)
1460		return -EINVAL;
1461
1462	bridge->ageing_time = clock_t_to_jiffies(ageing_time);
1463	return 0;
1464}
1465
1466int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
1467				       struct mlx5_esw_bridge_offloads *br_offloads)
1468{
1469	struct mlx5_esw_bridge *bridge;
1470	bool filtering;
1471
1472	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1473	if (!bridge)
1474		return -EINVAL;
1475
1476	filtering = bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG;
1477	if (filtering == enable)
1478		return 0;
1479
1480	mlx5_esw_bridge_fdb_flush(bridge);
1481	mlx5_esw_bridge_mdb_flush(bridge);
1482	if (enable)
1483		bridge->flags |= MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG;
1484	else
1485		bridge->flags &= ~MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG;
1486
1487	return 0;
1488}
1489
1490int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto,
1491				   struct mlx5_esw_bridge_offloads *br_offloads)
1492{
1493	struct mlx5_esw_bridge *bridge;
1494
1495	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id,
1496						  br_offloads);
1497	if (!bridge)
1498		return -EINVAL;
1499
1500	if (bridge->vlan_proto == proto)
1501		return 0;
1502	if (proto != ETH_P_8021Q && proto != ETH_P_8021AD) {
1503		esw_warn(br_offloads->esw->dev, "Can't set unsupported VLAN protocol %x", proto);
1504		return -EOPNOTSUPP;
1505	}
1506
1507	mlx5_esw_bridge_fdb_flush(bridge);
1508	mlx5_esw_bridge_mdb_flush(bridge);
1509	bridge->vlan_proto = proto;
1510	mlx5_esw_bridge_vlans_recreate(bridge);
1511
1512	return 0;
1513}
1514
1515int mlx5_esw_bridge_mcast_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable,
1516			      struct mlx5_esw_bridge_offloads *br_offloads)
1517{
1518	struct mlx5_eswitch *esw = br_offloads->esw;
1519	struct mlx5_esw_bridge *bridge;
1520	int err = 0;
1521	bool mcast;
1522
1523	if (!(MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_multi_path_any_table) ||
1524	      MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_multi_path_any_table_limit_regc)) ||
1525	    !MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_uplink_hairpin) ||
1526	    !MLX5_CAP_ESW_FLOWTABLE_FDB((esw)->dev, ignore_flow_level))
1527		return -EOPNOTSUPP;
1528
1529	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1530	if (!bridge)
1531		return -EINVAL;
1532
1533	mcast = bridge->flags & MLX5_ESW_BRIDGE_MCAST_FLAG;
1534	if (mcast == enable)
1535		return 0;
1536
1537	if (enable)
1538		err = mlx5_esw_bridge_mcast_enable(bridge);
1539	else
1540		mlx5_esw_bridge_mcast_disable(bridge);
1541
1542	return err;
1543}
1544
1545static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16 flags,
1546				      struct mlx5_esw_bridge_offloads *br_offloads,
1547				      struct mlx5_esw_bridge *bridge)
1548{
1549	struct mlx5_eswitch *esw = br_offloads->esw;
1550	struct mlx5_esw_bridge_port *port;
1551	int err;
1552
1553	port = kvzalloc(sizeof(*port), GFP_KERNEL);
1554	if (!port)
1555		return -ENOMEM;
1556
1557	port->vport_num = vport_num;
1558	port->esw_owner_vhca_id = esw_owner_vhca_id;
1559	port->bridge = bridge;
1560	port->flags |= flags;
1561	xa_init(&port->vlans);
1562
1563	err = mlx5_esw_bridge_port_mcast_init(port);
1564	if (err) {
1565		esw_warn(esw->dev,
1566			 "Failed to initialize port multicast (vport=%u,esw_owner_vhca_id=%u,err=%d)\n",
1567			 port->vport_num, port->esw_owner_vhca_id, err);
1568		goto err_port_mcast;
1569	}
1570
1571	err = mlx5_esw_bridge_port_insert(port, br_offloads);
1572	if (err) {
1573		esw_warn(esw->dev,
1574			 "Failed to insert port metadata (vport=%u,esw_owner_vhca_id=%u,err=%d)\n",
1575			 port->vport_num, port->esw_owner_vhca_id, err);
1576		goto err_port_insert;
1577	}
1578	trace_mlx5_esw_bridge_vport_init(port);
1579
1580	return 0;
1581
1582err_port_insert:
1583	mlx5_esw_bridge_port_mcast_cleanup(port);
1584err_port_mcast:
1585	kvfree(port);
1586	return err;
1587}
1588
1589static int mlx5_esw_bridge_vport_cleanup(struct mlx5_esw_bridge_offloads *br_offloads,
1590					 struct mlx5_esw_bridge_port *port)
1591{
1592	u16 vport_num = port->vport_num, esw_owner_vhca_id = port->esw_owner_vhca_id;
1593	struct mlx5_esw_bridge *bridge = port->bridge;
1594	struct mlx5_esw_bridge_fdb_entry *entry, *tmp;
1595
1596	list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list)
1597		if (entry->vport_num == vport_num && entry->esw_owner_vhca_id == esw_owner_vhca_id)
1598			mlx5_esw_bridge_fdb_entry_cleanup(entry, bridge);
1599
1600	trace_mlx5_esw_bridge_vport_cleanup(port);
1601	mlx5_esw_bridge_port_vlans_flush(port, bridge);
1602	mlx5_esw_bridge_port_mcast_cleanup(port);
1603	mlx5_esw_bridge_port_erase(port, br_offloads);
1604	kvfree(port);
1605	mlx5_esw_bridge_put(br_offloads, bridge);
1606	return 0;
1607}
1608
1609static int mlx5_esw_bridge_vport_link_with_flags(struct net_device *br_netdev, u16 vport_num,
1610						 u16 esw_owner_vhca_id, u16 flags,
1611						 struct mlx5_esw_bridge_offloads *br_offloads,
1612						 struct netlink_ext_ack *extack)
1613{
1614	struct mlx5_esw_bridge *bridge;
1615	int err;
1616
1617	bridge = mlx5_esw_bridge_lookup(br_netdev, br_offloads);
1618	if (IS_ERR(bridge)) {
1619		NL_SET_ERR_MSG_MOD(extack, "Error checking for existing bridge with same ifindex");
1620		return PTR_ERR(bridge);
1621	}
1622
1623	err = mlx5_esw_bridge_vport_init(vport_num, esw_owner_vhca_id, flags, br_offloads, bridge);
1624	if (err) {
1625		NL_SET_ERR_MSG_MOD(extack, "Error initializing port");
1626		goto err_vport;
1627	}
1628	return 0;
1629
1630err_vport:
1631	mlx5_esw_bridge_put(br_offloads, bridge);
1632	return err;
1633}
1634
1635int mlx5_esw_bridge_vport_link(struct net_device *br_netdev, u16 vport_num, u16 esw_owner_vhca_id,
1636			       struct mlx5_esw_bridge_offloads *br_offloads,
1637			       struct netlink_ext_ack *extack)
1638{
1639	return mlx5_esw_bridge_vport_link_with_flags(br_netdev, vport_num, esw_owner_vhca_id, 0,
1640						     br_offloads, extack);
1641}
1642
1643int mlx5_esw_bridge_vport_unlink(struct net_device *br_netdev, u16 vport_num,
1644				 u16 esw_owner_vhca_id,
1645				 struct mlx5_esw_bridge_offloads *br_offloads,
1646				 struct netlink_ext_ack *extack)
1647{
1648	struct mlx5_esw_bridge_port *port;
1649	int err;
1650
1651	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1652	if (!port) {
1653		NL_SET_ERR_MSG_MOD(extack, "Port is not attached to any bridge");
1654		return -EINVAL;
1655	}
1656	if (port->bridge->ifindex != br_netdev->ifindex) {
1657		NL_SET_ERR_MSG_MOD(extack, "Port is attached to another bridge");
1658		return -EINVAL;
1659	}
1660
1661	err = mlx5_esw_bridge_vport_cleanup(br_offloads, port);
1662	if (err)
1663		NL_SET_ERR_MSG_MOD(extack, "Port cleanup failed");
1664	return err;
1665}
1666
1667int mlx5_esw_bridge_vport_peer_link(struct net_device *br_netdev, u16 vport_num,
1668				    u16 esw_owner_vhca_id,
1669				    struct mlx5_esw_bridge_offloads *br_offloads,
1670				    struct netlink_ext_ack *extack)
1671{
1672	if (!MLX5_CAP_ESW(br_offloads->esw->dev, merged_eswitch))
1673		return 0;
1674
1675	return mlx5_esw_bridge_vport_link_with_flags(br_netdev, vport_num, esw_owner_vhca_id,
1676						     MLX5_ESW_BRIDGE_PORT_FLAG_PEER,
1677						     br_offloads, extack);
1678}
1679
1680int mlx5_esw_bridge_vport_peer_unlink(struct net_device *br_netdev, u16 vport_num,
1681				      u16 esw_owner_vhca_id,
1682				      struct mlx5_esw_bridge_offloads *br_offloads,
1683				      struct netlink_ext_ack *extack)
1684{
1685	return mlx5_esw_bridge_vport_unlink(br_netdev, vport_num, esw_owner_vhca_id, br_offloads,
1686					    extack);
1687}
1688
1689int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags,
1690				  struct mlx5_esw_bridge_offloads *br_offloads,
1691				  struct netlink_ext_ack *extack)
1692{
1693	struct mlx5_esw_bridge_port *port;
1694	struct mlx5_esw_bridge_vlan *vlan;
1695
1696	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1697	if (!port)
1698		return -EINVAL;
1699
1700	vlan = mlx5_esw_bridge_vlan_lookup(vid, port);
1701	if (vlan) {
1702		if (vlan->flags == flags)
1703			return 0;
1704		mlx5_esw_bridge_vlan_cleanup(port, vlan, port->bridge);
1705	}
1706
1707	vlan = mlx5_esw_bridge_vlan_create(port->bridge->vlan_proto, vid, flags, port,
1708					   br_offloads->esw);
1709	if (IS_ERR(vlan)) {
1710		NL_SET_ERR_MSG_MOD(extack, "Failed to create VLAN entry");
1711		return PTR_ERR(vlan);
1712	}
1713	return 0;
1714}
1715
1716void mlx5_esw_bridge_port_vlan_del(u16 vport_num, u16 esw_owner_vhca_id, u16 vid,
1717				   struct mlx5_esw_bridge_offloads *br_offloads)
1718{
1719	struct mlx5_esw_bridge_port *port;
1720	struct mlx5_esw_bridge_vlan *vlan;
1721
1722	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1723	if (!port)
1724		return;
1725
1726	vlan = mlx5_esw_bridge_vlan_lookup(vid, port);
1727	if (!vlan)
1728		return;
1729	mlx5_esw_bridge_vlan_cleanup(port, vlan, port->bridge);
1730}
1731
1732void mlx5_esw_bridge_fdb_update_used(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1733				     struct mlx5_esw_bridge_offloads *br_offloads,
1734				     struct switchdev_notifier_fdb_info *fdb_info)
1735{
1736	struct mlx5_esw_bridge_fdb_entry *entry;
1737	struct mlx5_esw_bridge *bridge;
1738
1739	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1740	if (!bridge)
1741		return;
1742
1743	entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid);
1744	if (!entry) {
1745		esw_debug(br_offloads->esw->dev,
1746			  "FDB update entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n",
1747			  fdb_info->addr, fdb_info->vid, vport_num);
1748		return;
1749	}
1750
1751	entry->lastuse = jiffies;
1752}
1753
1754void mlx5_esw_bridge_fdb_mark_deleted(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1755				      struct mlx5_esw_bridge_offloads *br_offloads,
1756				      struct switchdev_notifier_fdb_info *fdb_info)
1757{
1758	struct mlx5_esw_bridge_fdb_entry *entry;
1759	struct mlx5_esw_bridge *bridge;
1760
1761	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1762	if (!bridge)
1763		return;
1764
1765	entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid);
1766	if (!entry) {
1767		esw_debug(br_offloads->esw->dev,
1768			  "FDB mark deleted entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n",
1769			  fdb_info->addr, fdb_info->vid, vport_num);
1770		return;
1771	}
1772
1773	entry->flags |= MLX5_ESW_BRIDGE_FLAG_DELETED;
1774}
1775
1776void mlx5_esw_bridge_fdb_create(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1777				struct mlx5_esw_bridge_offloads *br_offloads,
1778				struct switchdev_notifier_fdb_info *fdb_info)
1779{
1780	struct mlx5_esw_bridge_fdb_entry *entry;
1781	struct mlx5_esw_bridge_port *port;
1782	struct mlx5_esw_bridge *bridge;
1783
1784	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1785	if (!port)
1786		return;
1787
1788	bridge = port->bridge;
1789	entry = mlx5_esw_bridge_fdb_entry_init(dev, vport_num, esw_owner_vhca_id, fdb_info->addr,
1790					       fdb_info->vid, fdb_info->added_by_user,
1791					       port->flags & MLX5_ESW_BRIDGE_PORT_FLAG_PEER,
1792					       br_offloads->esw, bridge);
1793	if (IS_ERR(entry))
1794		return;
1795
1796	if (entry->flags & MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER)
1797		mlx5_esw_bridge_fdb_offload_notify(dev, entry->key.addr, entry->key.vid,
1798						   SWITCHDEV_FDB_OFFLOADED);
1799	else if (!(entry->flags & MLX5_ESW_BRIDGE_FLAG_PEER))
1800		/* Take over dynamic entries to prevent kernel bridge from aging them out. */
1801		mlx5_esw_bridge_fdb_offload_notify(dev, entry->key.addr, entry->key.vid,
1802						   SWITCHDEV_FDB_ADD_TO_BRIDGE);
1803}
1804
1805void mlx5_esw_bridge_fdb_remove(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1806				struct mlx5_esw_bridge_offloads *br_offloads,
1807				struct switchdev_notifier_fdb_info *fdb_info)
1808{
1809	struct mlx5_eswitch *esw = br_offloads->esw;
1810	struct mlx5_esw_bridge_fdb_entry *entry;
1811	struct mlx5_esw_bridge *bridge;
1812
1813	bridge = mlx5_esw_bridge_from_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1814	if (!bridge)
1815		return;
1816
1817	entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid);
1818	if (!entry) {
1819		esw_debug(esw->dev,
1820			  "FDB remove entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n",
1821			  fdb_info->addr, fdb_info->vid, vport_num);
1822		return;
1823	}
1824
1825	mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
1826}
1827
1828void mlx5_esw_bridge_update(struct mlx5_esw_bridge_offloads *br_offloads)
1829{
1830	struct mlx5_esw_bridge_fdb_entry *entry, *tmp;
1831	struct mlx5_esw_bridge *bridge;
1832
1833	list_for_each_entry(bridge, &br_offloads->bridges, list) {
1834		list_for_each_entry_safe(entry, tmp, &bridge->fdb_list, list) {
1835			unsigned long lastuse =
1836				(unsigned long)mlx5_fc_query_lastuse(entry->ingress_counter);
1837
1838			if (entry->flags & (MLX5_ESW_BRIDGE_FLAG_ADDED_BY_USER |
1839					    MLX5_ESW_BRIDGE_FLAG_DELETED))
1840				continue;
1841
1842			if (time_after(lastuse, entry->lastuse))
1843				mlx5_esw_bridge_fdb_entry_refresh(entry);
1844			else if (!(entry->flags & MLX5_ESW_BRIDGE_FLAG_PEER) &&
1845				 time_is_before_jiffies(entry->lastuse + bridge->ageing_time))
1846				mlx5_esw_bridge_fdb_entry_notify_and_cleanup(entry, bridge);
1847		}
1848	}
1849}
1850
1851int mlx5_esw_bridge_port_mdb_add(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1852				 const unsigned char *addr, u16 vid,
1853				 struct mlx5_esw_bridge_offloads *br_offloads,
1854				 struct netlink_ext_ack *extack)
1855{
1856	struct mlx5_esw_bridge_vlan *vlan;
1857	struct mlx5_esw_bridge_port *port;
1858	struct mlx5_esw_bridge *bridge;
1859	int err;
1860
1861	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1862	if (!port) {
1863		esw_warn(br_offloads->esw->dev,
1864			 "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)\n",
1865			 addr, vport_num);
1866		NL_SET_ERR_MSG_FMT_MOD(extack,
1867				       "Failed to lookup bridge port to add MDB (MAC=%pM,vport=%u)",
1868				       addr, vport_num);
1869		return -EINVAL;
1870	}
1871
1872	bridge = port->bridge;
1873	if (bridge->flags & MLX5_ESW_BRIDGE_VLAN_FILTERING_FLAG && vid) {
1874		vlan = mlx5_esw_bridge_vlan_lookup(vid, port);
1875		if (!vlan) {
1876			esw_warn(br_offloads->esw->dev,
1877				 "Failed to lookup bridge port vlan metadata to create MDB (MAC=%pM,vid=%u,vport=%u)\n",
1878				 addr, vid, vport_num);
1879			NL_SET_ERR_MSG_FMT_MOD(extack,
1880					       "Failed to lookup vlan metadata for MDB (MAC=%pM,vid=%u,vport=%u)",
1881					       addr, vid, vport_num);
1882			return -EINVAL;
1883		}
1884	}
1885
1886	err = mlx5_esw_bridge_port_mdb_attach(dev, port, addr, vid);
1887	if (err) {
1888		NL_SET_ERR_MSG_FMT_MOD(extack, "Failed to add MDB (MAC=%pM,vid=%u,vport=%u)",
1889				       addr, vid, vport_num);
1890		return err;
1891	}
1892
1893	return 0;
1894}
1895
1896void mlx5_esw_bridge_port_mdb_del(struct net_device *dev, u16 vport_num, u16 esw_owner_vhca_id,
1897				  const unsigned char *addr, u16 vid,
1898				  struct mlx5_esw_bridge_offloads *br_offloads)
1899{
1900	struct mlx5_esw_bridge_port *port;
1901
1902	port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, br_offloads);
1903	if (!port)
1904		return;
1905
1906	mlx5_esw_bridge_port_mdb_detach(dev, port, addr, vid);
1907}
1908
1909static void mlx5_esw_bridge_flush(struct mlx5_esw_bridge_offloads *br_offloads)
1910{
1911	struct mlx5_esw_bridge_port *port;
1912	unsigned long i;
1913
1914	xa_for_each(&br_offloads->ports, i, port)
1915		mlx5_esw_bridge_vport_cleanup(br_offloads, port);
1916
1917	WARN_ONCE(!list_empty(&br_offloads->bridges),
1918		  "Cleaning up bridge offloads while still having bridges attached\n");
1919}
1920
1921struct mlx5_esw_bridge_offloads *mlx5_esw_bridge_init(struct mlx5_eswitch *esw)
1922{
1923	struct mlx5_esw_bridge_offloads *br_offloads;
1924
1925	ASSERT_RTNL();
1926
1927	br_offloads = kvzalloc(sizeof(*br_offloads), GFP_KERNEL);
1928	if (!br_offloads)
1929		return ERR_PTR(-ENOMEM);
1930
1931	INIT_LIST_HEAD(&br_offloads->bridges);
1932	xa_init(&br_offloads->ports);
1933	br_offloads->esw = esw;
1934	esw->br_offloads = br_offloads;
1935	mlx5_esw_bridge_debugfs_offloads_init(br_offloads);
1936
1937	return br_offloads;
1938}
1939
1940void mlx5_esw_bridge_cleanup(struct mlx5_eswitch *esw)
1941{
1942	struct mlx5_esw_bridge_offloads *br_offloads = esw->br_offloads;
1943
1944	ASSERT_RTNL();
1945
1946	if (!br_offloads)
1947		return;
1948
1949	mlx5_esw_bridge_flush(br_offloads);
1950	WARN_ON(!xa_empty(&br_offloads->ports));
1951	mlx5_esw_bridge_debugfs_offloads_cleanup(br_offloads);
1952
1953	esw->br_offloads = NULL;
1954	kvfree(br_offloads);
1955}