eswitch.c

   1/*
   2 * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/etherdevice.h>
  34#include <linux/debugfs.h>
  35#include <linux/mlx5/driver.h>
  36#include <linux/mlx5/mlx5_ifc.h>
  37#include <linux/mlx5/vport.h>
  38#include <linux/mlx5/fs.h>
  39#include <linux/mlx5/mpfs.h>
  40#include "esw/acl/lgcy.h"
  41#include "esw/legacy.h"
  42#include "esw/qos.h"
  43#include "mlx5_core.h"
  44#include "lib/eq.h"
  45#include "lag/lag.h"
  46#include "eswitch.h"
  47#include "fs_core.h"
  48#include "devlink.h"
  49#include "ecpf.h"
  50#include "en/mod_hdr.h"
  51#include "en_accel/ipsec.h"
  52
  53enum {
  54	MLX5_ACTION_NONE = 0,
  55	MLX5_ACTION_ADD  = 1,
  56	MLX5_ACTION_DEL  = 2,
  57};
  58
  59/* Vport UC/MC hash node */
  60struct vport_addr {
  61	struct l2addr_node     node;
  62	u8                     action;
  63	u16                    vport;
  64	struct mlx5_flow_handle *flow_rule;
  65	bool mpfs; /* UC MAC was added to MPFs */
  66	/* A flag indicating that mac was added due to mc promiscuous vport */
  67	bool mc_promisc;
  68};
  69
  70static int mlx5_eswitch_check(const struct mlx5_core_dev *dev)
  71{
  72	if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
  73		return -EOPNOTSUPP;
  74
  75	if (!MLX5_ESWITCH_MANAGER(dev))
  76		return -EOPNOTSUPP;
  77
  78	return 0;
  79}
  80
  81static struct mlx5_eswitch *__mlx5_devlink_eswitch_get(struct devlink *devlink, bool check)
  82{
  83	struct mlx5_core_dev *dev = devlink_priv(devlink);
  84	int err;
  85
  86	if (check) {
  87		err = mlx5_eswitch_check(dev);
  88		if (err)
  89			return ERR_PTR(err);
  90	}
  91
  92	return dev->priv.eswitch;
  93}
  94
  95struct mlx5_eswitch *__must_check
  96mlx5_devlink_eswitch_get(struct devlink *devlink)
  97{
  98	return __mlx5_devlink_eswitch_get(devlink, true);
  99}
 100
 101struct mlx5_eswitch *mlx5_devlink_eswitch_nocheck_get(struct devlink *devlink)
 102{
 103	return __mlx5_devlink_eswitch_get(devlink, false);
 104}
 105
 106struct mlx5_vport *__must_check
 107mlx5_eswitch_get_vport(struct mlx5_eswitch *esw, u16 vport_num)
 108{
 109	struct mlx5_vport *vport;
 110
 111	if (!esw)
 112		return ERR_PTR(-EPERM);
 113
 114	vport = xa_load(&esw->vports, vport_num);
 115	if (!vport) {
 116		esw_debug(esw->dev, "vport out of range: num(0x%x)\n", vport_num);
 117		return ERR_PTR(-EINVAL);
 118	}
 119	return vport;
 120}
 121
 122static int arm_vport_context_events_cmd(struct mlx5_core_dev *dev, u16 vport,
 123					u32 events_mask)
 124{
 125	u32 in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)] = {};
 126	void *nic_vport_ctx;
 127
 128	MLX5_SET(modify_nic_vport_context_in, in,
 129		 opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
 130	MLX5_SET(modify_nic_vport_context_in, in, field_select.change_event, 1);
 131	MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport);
 132	if (vport || mlx5_core_is_ecpf(dev))
 133		MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1);
 134	nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in,
 135				     in, nic_vport_context);
 136
 137	MLX5_SET(nic_vport_context, nic_vport_ctx, arm_change_event, 1);
 138
 139	if (events_mask & MLX5_VPORT_UC_ADDR_CHANGE)
 140		MLX5_SET(nic_vport_context, nic_vport_ctx,
 141			 event_on_uc_address_change, 1);
 142	if (events_mask & MLX5_VPORT_MC_ADDR_CHANGE)
 143		MLX5_SET(nic_vport_context, nic_vport_ctx,
 144			 event_on_mc_address_change, 1);
 145	if (events_mask & MLX5_VPORT_PROMISC_CHANGE)
 146		MLX5_SET(nic_vport_context, nic_vport_ctx,
 147			 event_on_promisc_change, 1);
 148
 149	return mlx5_cmd_exec_in(dev, modify_nic_vport_context, in);
 150}
 151
 152/* E-Switch vport context HW commands */
 153int mlx5_eswitch_modify_esw_vport_context(struct mlx5_core_dev *dev, u16 vport,
 154					  bool other_vport, void *in)
 155{
 156	MLX5_SET(modify_esw_vport_context_in, in, opcode,
 157		 MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT);
 158	MLX5_SET(modify_esw_vport_context_in, in, vport_number, vport);
 159	MLX5_SET(modify_esw_vport_context_in, in, other_vport, other_vport);
 160	return mlx5_cmd_exec_in(dev, modify_esw_vport_context, in);
 161}
 162
 163static int modify_esw_vport_cvlan(struct mlx5_core_dev *dev, u16 vport,
 164				  u16 vlan, u8 qos, u8 set_flags)
 165{
 166	u32 in[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
 167
 168	if (!MLX5_CAP_ESW(dev, vport_cvlan_strip) ||
 169	    !MLX5_CAP_ESW(dev, vport_cvlan_insert_if_not_exist))
 170		return -EOPNOTSUPP;
 171
 172	esw_debug(dev, "Set Vport[%d] VLAN %d qos %d set=%x\n",
 173		  vport, vlan, qos, set_flags);
 174
 175	if (set_flags & SET_VLAN_STRIP)
 176		MLX5_SET(modify_esw_vport_context_in, in,
 177			 esw_vport_context.vport_cvlan_strip, 1);
 178
 179	if (set_flags & SET_VLAN_INSERT) {
 180		if (MLX5_CAP_ESW(dev, vport_cvlan_insert_always)) {
 181			/* insert either if vlan exist in packet or not */
 182			MLX5_SET(modify_esw_vport_context_in, in,
 183				 esw_vport_context.vport_cvlan_insert,
 184				 MLX5_VPORT_CVLAN_INSERT_ALWAYS);
 185		} else {
 186			/* insert only if no vlan in packet */
 187			MLX5_SET(modify_esw_vport_context_in, in,
 188				 esw_vport_context.vport_cvlan_insert,
 189				 MLX5_VPORT_CVLAN_INSERT_WHEN_NO_CVLAN);
 190		}
 191		MLX5_SET(modify_esw_vport_context_in, in,
 192			 esw_vport_context.cvlan_pcp, qos);
 193		MLX5_SET(modify_esw_vport_context_in, in,
 194			 esw_vport_context.cvlan_id, vlan);
 195	}
 196
 197	MLX5_SET(modify_esw_vport_context_in, in,
 198		 field_select.vport_cvlan_strip, 1);
 199	MLX5_SET(modify_esw_vport_context_in, in,
 200		 field_select.vport_cvlan_insert, 1);
 201
 202	return mlx5_eswitch_modify_esw_vport_context(dev, vport, true, in);
 203}
 204
 205/* E-Switch FDB */
 206static struct mlx5_flow_handle *
 207__esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u16 vport, bool rx_rule,
 208			 u8 mac_c[ETH_ALEN], u8 mac_v[ETH_ALEN])
 209{
 210	int match_header = (is_zero_ether_addr(mac_c) ? 0 :
 211			    MLX5_MATCH_OUTER_HEADERS);
 212	struct mlx5_flow_handle *flow_rule = NULL;
 213	struct mlx5_flow_act flow_act = {0};
 214	struct mlx5_flow_destination dest = {};
 215	struct mlx5_flow_spec *spec;
 216	void *mv_misc = NULL;
 217	void *mc_misc = NULL;
 218	u8 *dmac_v = NULL;
 219	u8 *dmac_c = NULL;
 220
 221	if (rx_rule)
 222		match_header |= MLX5_MATCH_MISC_PARAMETERS;
 223
 224	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 225	if (!spec)
 226		return NULL;
 227
 228	dmac_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 229			      outer_headers.dmac_47_16);
 230	dmac_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 231			      outer_headers.dmac_47_16);
 232
 233	if (match_header & MLX5_MATCH_OUTER_HEADERS) {
 234		ether_addr_copy(dmac_v, mac_v);
 235		ether_addr_copy(dmac_c, mac_c);
 236	}
 237
 238	if (match_header & MLX5_MATCH_MISC_PARAMETERS) {
 239		mv_misc  = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 240					misc_parameters);
 241		mc_misc  = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 242					misc_parameters);
 243		MLX5_SET(fte_match_set_misc, mv_misc, source_port, MLX5_VPORT_UPLINK);
 244		MLX5_SET_TO_ONES(fte_match_set_misc, mc_misc, source_port);
 245	}
 246
 247	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 248	dest.vport.num = vport;
 249
 250	esw_debug(esw->dev,
 251		  "\tFDB add rule dmac_v(%pM) dmac_c(%pM) -> vport(%d)\n",
 252		  dmac_v, dmac_c, vport);
 253	spec->match_criteria_enable = match_header;
 254	flow_act.action =  MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 255	flow_rule =
 256		mlx5_add_flow_rules(esw->fdb_table.legacy.fdb, spec,
 257				    &flow_act, &dest, 1);
 258	if (IS_ERR(flow_rule)) {
 259		esw_warn(esw->dev,
 260			 "FDB: Failed to add flow rule: dmac_v(%pM) dmac_c(%pM) -> vport(%d), err(%pe)\n",
 261			 dmac_v, dmac_c, vport, flow_rule);
 262		flow_rule = NULL;
 263	}
 264
 265	kvfree(spec);
 266	return flow_rule;
 267}
 268
 269static struct mlx5_flow_handle *
 270esw_fdb_set_vport_rule(struct mlx5_eswitch *esw, u8 mac[ETH_ALEN], u16 vport)
 271{
 272	u8 mac_c[ETH_ALEN];
 273
 274	eth_broadcast_addr(mac_c);
 275	return __esw_fdb_set_vport_rule(esw, vport, false, mac_c, mac);
 276}
 277
 278static struct mlx5_flow_handle *
 279esw_fdb_set_vport_allmulti_rule(struct mlx5_eswitch *esw, u16 vport)
 280{
 281	u8 mac_c[ETH_ALEN];
 282	u8 mac_v[ETH_ALEN];
 283
 284	eth_zero_addr(mac_c);
 285	eth_zero_addr(mac_v);
 286	mac_c[0] = 0x01;
 287	mac_v[0] = 0x01;
 288	return __esw_fdb_set_vport_rule(esw, vport, false, mac_c, mac_v);
 289}
 290
 291static struct mlx5_flow_handle *
 292esw_fdb_set_vport_promisc_rule(struct mlx5_eswitch *esw, u16 vport)
 293{
 294	u8 mac_c[ETH_ALEN];
 295	u8 mac_v[ETH_ALEN];
 296
 297	eth_zero_addr(mac_c);
 298	eth_zero_addr(mac_v);
 299	return __esw_fdb_set_vport_rule(esw, vport, true, mac_c, mac_v);
 300}
 301
 302/* E-Switch vport UC/MC lists management */
 303typedef int (*vport_addr_action)(struct mlx5_eswitch *esw,
 304				 struct vport_addr *vaddr);
 305
 306static int esw_add_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 307{
 308	u8 *mac = vaddr->node.addr;
 309	u16 vport = vaddr->vport;
 310	int err;
 311
 312	/* Skip mlx5_mpfs_add_mac for eswitch_managers,
 313	 * it is already done by its netdev in mlx5e_execute_l2_action
 314	 */
 315	if (mlx5_esw_is_manager_vport(esw, vport))
 316		goto fdb_add;
 317
 318	err = mlx5_mpfs_add_mac(esw->dev, mac);
 319	if (err) {
 320		esw_warn(esw->dev,
 321			 "Failed to add L2 table mac(%pM) for vport(0x%x), err(%d)\n",
 322			 mac, vport, err);
 323		return err;
 324	}
 325	vaddr->mpfs = true;
 326
 327fdb_add:
 328	/* SRIOV is enabled: Forward UC MAC to vport */
 329	if (esw->fdb_table.legacy.fdb && esw->mode == MLX5_ESWITCH_LEGACY) {
 330		vaddr->flow_rule = esw_fdb_set_vport_rule(esw, mac, vport);
 331
 332		esw_debug(esw->dev, "\tADDED UC MAC: vport[%d] %pM fr(%p)\n",
 333			  vport, mac, vaddr->flow_rule);
 334	}
 335
 336	return 0;
 337}
 338
 339static int esw_del_uc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 340{
 341	u8 *mac = vaddr->node.addr;
 342	u16 vport = vaddr->vport;
 343	int err = 0;
 344
 345	/* Skip mlx5_mpfs_del_mac for eswitch managers,
 346	 * it is already done by its netdev in mlx5e_execute_l2_action
 347	 */
 348	if (!vaddr->mpfs || mlx5_esw_is_manager_vport(esw, vport))
 349		goto fdb_del;
 350
 351	err = mlx5_mpfs_del_mac(esw->dev, mac);
 352	if (err)
 353		esw_warn(esw->dev,
 354			 "Failed to del L2 table mac(%pM) for vport(%d), err(%d)\n",
 355			 mac, vport, err);
 356	vaddr->mpfs = false;
 357
 358fdb_del:
 359	if (vaddr->flow_rule)
 360		mlx5_del_flow_rules(vaddr->flow_rule);
 361	vaddr->flow_rule = NULL;
 362
 363	return 0;
 364}
 365
 366static void update_allmulti_vports(struct mlx5_eswitch *esw,
 367				   struct vport_addr *vaddr,
 368				   struct esw_mc_addr *esw_mc)
 369{
 370	u8 *mac = vaddr->node.addr;
 371	struct mlx5_vport *vport;
 372	unsigned long i;
 373	u16 vport_num;
 374
 375	mlx5_esw_for_each_vport(esw, i, vport) {
 376		struct hlist_head *vport_hash = vport->mc_list;
 377		struct vport_addr *iter_vaddr =
 378					l2addr_hash_find(vport_hash,
 379							 mac,
 380							 struct vport_addr);
 381		vport_num = vport->vport;
 382		if (IS_ERR_OR_NULL(vport->allmulti_rule) ||
 383		    vaddr->vport == vport_num)
 384			continue;
 385		switch (vaddr->action) {
 386		case MLX5_ACTION_ADD:
 387			if (iter_vaddr)
 388				continue;
 389			iter_vaddr = l2addr_hash_add(vport_hash, mac,
 390						     struct vport_addr,
 391						     GFP_KERNEL);
 392			if (!iter_vaddr) {
 393				esw_warn(esw->dev,
 394					 "ALL-MULTI: Failed to add MAC(%pM) to vport[%d] DB\n",
 395					 mac, vport_num);
 396				continue;
 397			}
 398			iter_vaddr->vport = vport_num;
 399			iter_vaddr->flow_rule =
 400					esw_fdb_set_vport_rule(esw,
 401							       mac,
 402							       vport_num);
 403			iter_vaddr->mc_promisc = true;
 404			break;
 405		case MLX5_ACTION_DEL:
 406			if (!iter_vaddr)
 407				continue;
 408			mlx5_del_flow_rules(iter_vaddr->flow_rule);
 409			l2addr_hash_del(iter_vaddr);
 410			break;
 411		}
 412	}
 413}
 414
 415static int esw_add_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 416{
 417	struct hlist_head *hash = esw->mc_table;
 418	struct esw_mc_addr *esw_mc;
 419	u8 *mac = vaddr->node.addr;
 420	u16 vport = vaddr->vport;
 421
 422	if (!esw->fdb_table.legacy.fdb)
 423		return 0;
 424
 425	esw_mc = l2addr_hash_find(hash, mac, struct esw_mc_addr);
 426	if (esw_mc)
 427		goto add;
 428
 429	esw_mc = l2addr_hash_add(hash, mac, struct esw_mc_addr, GFP_KERNEL);
 430	if (!esw_mc)
 431		return -ENOMEM;
 432
 433	esw_mc->uplink_rule = /* Forward MC MAC to Uplink */
 434		esw_fdb_set_vport_rule(esw, mac, MLX5_VPORT_UPLINK);
 435
 436	/* Add this multicast mac to all the mc promiscuous vports */
 437	update_allmulti_vports(esw, vaddr, esw_mc);
 438
 439add:
 440	/* If the multicast mac is added as a result of mc promiscuous vport,
 441	 * don't increment the multicast ref count
 442	 */
 443	if (!vaddr->mc_promisc)
 444		esw_mc->refcnt++;
 445
 446	/* Forward MC MAC to vport */
 447	vaddr->flow_rule = esw_fdb_set_vport_rule(esw, mac, vport);
 448	esw_debug(esw->dev,
 449		  "\tADDED MC MAC: vport[%d] %pM fr(%p) refcnt(%d) uplinkfr(%p)\n",
 450		  vport, mac, vaddr->flow_rule,
 451		  esw_mc->refcnt, esw_mc->uplink_rule);
 452	return 0;
 453}
 454
 455static int esw_del_mc_addr(struct mlx5_eswitch *esw, struct vport_addr *vaddr)
 456{
 457	struct hlist_head *hash = esw->mc_table;
 458	struct esw_mc_addr *esw_mc;
 459	u8 *mac = vaddr->node.addr;
 460	u16 vport = vaddr->vport;
 461
 462	if (!esw->fdb_table.legacy.fdb)
 463		return 0;
 464
 465	esw_mc = l2addr_hash_find(hash, mac, struct esw_mc_addr);
 466	if (!esw_mc) {
 467		esw_warn(esw->dev,
 468			 "Failed to find eswitch MC addr for MAC(%pM) vport(%d)",
 469			 mac, vport);
 470		return -EINVAL;
 471	}
 472	esw_debug(esw->dev,
 473		  "\tDELETE MC MAC: vport[%d] %pM fr(%p) refcnt(%d) uplinkfr(%p)\n",
 474		  vport, mac, vaddr->flow_rule, esw_mc->refcnt,
 475		  esw_mc->uplink_rule);
 476
 477	if (vaddr->flow_rule)
 478		mlx5_del_flow_rules(vaddr->flow_rule);
 479	vaddr->flow_rule = NULL;
 480
 481	/* If the multicast mac is added as a result of mc promiscuous vport,
 482	 * don't decrement the multicast ref count.
 483	 */
 484	if (vaddr->mc_promisc || (--esw_mc->refcnt > 0))
 485		return 0;
 486
 487	/* Remove this multicast mac from all the mc promiscuous vports */
 488	update_allmulti_vports(esw, vaddr, esw_mc);
 489
 490	if (esw_mc->uplink_rule)
 491		mlx5_del_flow_rules(esw_mc->uplink_rule);
 492
 493	l2addr_hash_del(esw_mc);
 494	return 0;
 495}
 496
 497/* Apply vport UC/MC list to HW l2 table and FDB table */
 498static void esw_apply_vport_addr_list(struct mlx5_eswitch *esw,
 499				      struct mlx5_vport *vport, int list_type)
 500{
 501	bool is_uc = list_type == MLX5_NVPRT_LIST_TYPE_UC;
 502	vport_addr_action vport_addr_add;
 503	vport_addr_action vport_addr_del;
 504	struct vport_addr *addr;
 505	struct l2addr_node *node;
 506	struct hlist_head *hash;
 507	struct hlist_node *tmp;
 508	int hi;
 509
 510	vport_addr_add = is_uc ? esw_add_uc_addr :
 511				 esw_add_mc_addr;
 512	vport_addr_del = is_uc ? esw_del_uc_addr :
 513				 esw_del_mc_addr;
 514
 515	hash = is_uc ? vport->uc_list : vport->mc_list;
 516	for_each_l2hash_node(node, tmp, hash, hi) {
 517		addr = container_of(node, struct vport_addr, node);
 518		switch (addr->action) {
 519		case MLX5_ACTION_ADD:
 520			vport_addr_add(esw, addr);
 521			addr->action = MLX5_ACTION_NONE;
 522			break;
 523		case MLX5_ACTION_DEL:
 524			vport_addr_del(esw, addr);
 525			l2addr_hash_del(addr);
 526			break;
 527		}
 528	}
 529}
 530
 531/* Sync vport UC/MC list from vport context */
 532static void esw_update_vport_addr_list(struct mlx5_eswitch *esw,
 533				       struct mlx5_vport *vport, int list_type)
 534{
 535	bool is_uc = list_type == MLX5_NVPRT_LIST_TYPE_UC;
 536	u8 (*mac_list)[ETH_ALEN];
 537	struct l2addr_node *node;
 538	struct vport_addr *addr;
 539	struct hlist_head *hash;
 540	struct hlist_node *tmp;
 541	int size;
 542	int err;
 543	int hi;
 544	int i;
 545
 546	size = is_uc ? MLX5_MAX_UC_PER_VPORT(esw->dev) :
 547		       MLX5_MAX_MC_PER_VPORT(esw->dev);
 548
 549	mac_list = kcalloc(size, ETH_ALEN, GFP_KERNEL);
 550	if (!mac_list)
 551		return;
 552
 553	hash = is_uc ? vport->uc_list : vport->mc_list;
 554
 555	for_each_l2hash_node(node, tmp, hash, hi) {
 556		addr = container_of(node, struct vport_addr, node);
 557		addr->action = MLX5_ACTION_DEL;
 558	}
 559
 560	if (!vport->enabled)
 561		goto out;
 562
 563	err = mlx5_query_nic_vport_mac_list(esw->dev, vport->vport, list_type,
 564					    mac_list, &size);
 565	if (err)
 566		goto out;
 567	esw_debug(esw->dev, "vport[%d] context update %s list size (%d)\n",
 568		  vport->vport, is_uc ? "UC" : "MC", size);
 569
 570	for (i = 0; i < size; i++) {
 571		if (is_uc && !is_valid_ether_addr(mac_list[i]))
 572			continue;
 573
 574		if (!is_uc && !is_multicast_ether_addr(mac_list[i]))
 575			continue;
 576
 577		addr = l2addr_hash_find(hash, mac_list[i], struct vport_addr);
 578		if (addr) {
 579			addr->action = MLX5_ACTION_NONE;
 580			/* If this mac was previously added because of allmulti
 581			 * promiscuous rx mode, its now converted to be original
 582			 * vport mac.
 583			 */
 584			if (addr->mc_promisc) {
 585				struct esw_mc_addr *esw_mc =
 586					l2addr_hash_find(esw->mc_table,
 587							 mac_list[i],
 588							 struct esw_mc_addr);
 589				if (!esw_mc) {
 590					esw_warn(esw->dev,
 591						 "Failed to MAC(%pM) in mcast DB\n",
 592						 mac_list[i]);
 593					continue;
 594				}
 595				esw_mc->refcnt++;
 596				addr->mc_promisc = false;
 597			}
 598			continue;
 599		}
 600
 601		addr = l2addr_hash_add(hash, mac_list[i], struct vport_addr,
 602				       GFP_KERNEL);
 603		if (!addr) {
 604			esw_warn(esw->dev,
 605				 "Failed to add MAC(%pM) to vport[%d] DB\n",
 606				 mac_list[i], vport->vport);
 607			continue;
 608		}
 609		addr->vport = vport->vport;
 610		addr->action = MLX5_ACTION_ADD;
 611	}
 612out:
 613	kfree(mac_list);
 614}
 615
 616/* Sync vport UC/MC list from vport context
 617 * Must be called after esw_update_vport_addr_list
 618 */
 619static void esw_update_vport_mc_promisc(struct mlx5_eswitch *esw,
 620					struct mlx5_vport *vport)
 621{
 622	struct l2addr_node *node;
 623	struct vport_addr *addr;
 624	struct hlist_head *hash;
 625	struct hlist_node *tmp;
 626	int hi;
 627
 628	hash = vport->mc_list;
 629
 630	for_each_l2hash_node(node, tmp, esw->mc_table, hi) {
 631		u8 *mac = node->addr;
 632
 633		addr = l2addr_hash_find(hash, mac, struct vport_addr);
 634		if (addr) {
 635			if (addr->action == MLX5_ACTION_DEL)
 636				addr->action = MLX5_ACTION_NONE;
 637			continue;
 638		}
 639		addr = l2addr_hash_add(hash, mac, struct vport_addr,
 640				       GFP_KERNEL);
 641		if (!addr) {
 642			esw_warn(esw->dev,
 643				 "Failed to add allmulti MAC(%pM) to vport[%d] DB\n",
 644				 mac, vport->vport);
 645			continue;
 646		}
 647		addr->vport = vport->vport;
 648		addr->action = MLX5_ACTION_ADD;
 649		addr->mc_promisc = true;
 650	}
 651}
 652
 653/* Apply vport rx mode to HW FDB table */
 654static void esw_apply_vport_rx_mode(struct mlx5_eswitch *esw,
 655				    struct mlx5_vport *vport,
 656				    bool promisc, bool mc_promisc)
 657{
 658	struct esw_mc_addr *allmulti_addr = &esw->mc_promisc;
 659
 660	if (IS_ERR_OR_NULL(vport->allmulti_rule) != mc_promisc)
 661		goto promisc;
 662
 663	if (mc_promisc) {
 664		vport->allmulti_rule =
 665			esw_fdb_set_vport_allmulti_rule(esw, vport->vport);
 666		if (!allmulti_addr->uplink_rule)
 667			allmulti_addr->uplink_rule =
 668				esw_fdb_set_vport_allmulti_rule(esw,
 669								MLX5_VPORT_UPLINK);
 670		allmulti_addr->refcnt++;
 671	} else if (vport->allmulti_rule) {
 672		mlx5_del_flow_rules(vport->allmulti_rule);
 673		vport->allmulti_rule = NULL;
 674
 675		if (--allmulti_addr->refcnt > 0)
 676			goto promisc;
 677
 678		if (allmulti_addr->uplink_rule)
 679			mlx5_del_flow_rules(allmulti_addr->uplink_rule);
 680		allmulti_addr->uplink_rule = NULL;
 681	}
 682
 683promisc:
 684	if (IS_ERR_OR_NULL(vport->promisc_rule) != promisc)
 685		return;
 686
 687	if (promisc) {
 688		vport->promisc_rule =
 689			esw_fdb_set_vport_promisc_rule(esw, vport->vport);
 690	} else if (vport->promisc_rule) {
 691		mlx5_del_flow_rules(vport->promisc_rule);
 692		vport->promisc_rule = NULL;
 693	}
 694}
 695
 696/* Sync vport rx mode from vport context */
 697static void esw_update_vport_rx_mode(struct mlx5_eswitch *esw,
 698				     struct mlx5_vport *vport)
 699{
 700	int promisc_all = 0;
 701	int promisc_uc = 0;
 702	int promisc_mc = 0;
 703	int err;
 704
 705	err = mlx5_query_nic_vport_promisc(esw->dev,
 706					   vport->vport,
 707					   &promisc_uc,
 708					   &promisc_mc,
 709					   &promisc_all);
 710	if (err)
 711		return;
 712	esw_debug(esw->dev, "vport[%d] context update rx mode promisc_all=%d, all_multi=%d\n",
 713		  vport->vport, promisc_all, promisc_mc);
 714
 715	if (!vport->info.trusted || !vport->enabled) {
 716		promisc_uc = 0;
 717		promisc_mc = 0;
 718		promisc_all = 0;
 719	}
 720
 721	esw_apply_vport_rx_mode(esw, vport, promisc_all,
 722				(promisc_all || promisc_mc));
 723}
 724
 725void esw_vport_change_handle_locked(struct mlx5_vport *vport)
 726{
 727	struct mlx5_core_dev *dev = vport->dev;
 728	struct mlx5_eswitch *esw = dev->priv.eswitch;
 729	u8 mac[ETH_ALEN];
 730
 731	if (!MLX5_CAP_GEN(dev, log_max_l2_table))
 732		return;
 733
 734	mlx5_query_nic_vport_mac_address(dev, vport->vport, true, mac);
 735	esw_debug(dev, "vport[%d] Context Changed: perm mac: %pM\n",
 736		  vport->vport, mac);
 737
 738	if (vport->enabled_events & MLX5_VPORT_UC_ADDR_CHANGE) {
 739		esw_update_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_UC);
 740		esw_apply_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_UC);
 741	}
 742
 743	if (vport->enabled_events & MLX5_VPORT_MC_ADDR_CHANGE)
 744		esw_update_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_MC);
 745
 746	if (vport->enabled_events & MLX5_VPORT_PROMISC_CHANGE) {
 747		esw_update_vport_rx_mode(esw, vport);
 748		if (!IS_ERR_OR_NULL(vport->allmulti_rule))
 749			esw_update_vport_mc_promisc(esw, vport);
 750	}
 751
 752	if (vport->enabled_events & (MLX5_VPORT_PROMISC_CHANGE | MLX5_VPORT_MC_ADDR_CHANGE))
 753		esw_apply_vport_addr_list(esw, vport, MLX5_NVPRT_LIST_TYPE_MC);
 754
 755	esw_debug(esw->dev, "vport[%d] Context Changed: Done\n", vport->vport);
 756	if (vport->enabled)
 757		arm_vport_context_events_cmd(dev, vport->vport,
 758					     vport->enabled_events);
 759}
 760
 761static void esw_vport_change_handler(struct work_struct *work)
 762{
 763	struct mlx5_vport *vport =
 764		container_of(work, struct mlx5_vport, vport_change_handler);
 765	struct mlx5_eswitch *esw = vport->dev->priv.eswitch;
 766
 767	mutex_lock(&esw->state_lock);
 768	esw_vport_change_handle_locked(vport);
 769	mutex_unlock(&esw->state_lock);
 770}
 771
 772static void node_guid_gen_from_mac(u64 *node_guid, const u8 *mac)
 773{
 774	((u8 *)node_guid)[7] = mac[0];
 775	((u8 *)node_guid)[6] = mac[1];
 776	((u8 *)node_guid)[5] = mac[2];
 777	((u8 *)node_guid)[4] = 0xff;
 778	((u8 *)node_guid)[3] = 0xfe;
 779	((u8 *)node_guid)[2] = mac[3];
 780	((u8 *)node_guid)[1] = mac[4];
 781	((u8 *)node_guid)[0] = mac[5];
 782}
 783
 784static int esw_vport_setup_acl(struct mlx5_eswitch *esw,
 785			       struct mlx5_vport *vport)
 786{
 787	if (esw->mode == MLX5_ESWITCH_LEGACY)
 788		return esw_legacy_vport_acl_setup(esw, vport);
 789	else
 790		return esw_vport_create_offloads_acl_tables(esw, vport);
 791}
 792
 793static void esw_vport_cleanup_acl(struct mlx5_eswitch *esw,
 794				  struct mlx5_vport *vport)
 795{
 796	if (esw->mode == MLX5_ESWITCH_LEGACY)
 797		esw_legacy_vport_acl_cleanup(esw, vport);
 798	else
 799		esw_vport_destroy_offloads_acl_tables(esw, vport);
 800}
 801
 802static int mlx5_esw_vport_caps_get(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 803{
 804	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
 805	void *query_ctx;
 806	void *hca_caps;
 807	int err;
 808
 809	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
 810		return 0;
 811
 812	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
 813	if (!query_ctx)
 814		return -ENOMEM;
 815
 816	err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
 817					    MLX5_CAP_GENERAL);
 818	if (err)
 819		goto out_free;
 820
 821	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
 822	vport->info.roce_enabled = MLX5_GET(cmd_hca_cap, hca_caps, roce);
 823	vport->vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
 824
 825	if (!MLX5_CAP_GEN_MAX(esw->dev, hca_cap_2))
 826		goto out_free;
 827
 828	memset(query_ctx, 0, query_out_sz);
 829	err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx,
 830					    MLX5_CAP_GENERAL_2);
 831	if (err)
 832		goto out_free;
 833
 834	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
 835	vport->info.mig_enabled = MLX5_GET(cmd_hca_cap_2, hca_caps, migratable);
 836
 837	err = mlx5_esw_ipsec_vf_offload_get(esw->dev, vport);
 838out_free:
 839	kfree(query_ctx);
 840	return err;
 841}
 842
 843bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id)
 844{
 845	struct mlx5_vport *vport;
 846
 847	vport = mlx5_eswitch_get_vport(esw, vportn);
 848	if (IS_ERR(vport) || MLX5_VPORT_INVAL_VHCA_ID(vport))
 849		return false;
 850
 851	*vhca_id = vport->vhca_id;
 852	return true;
 853}
 854
 855static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 856{
 857	bool vst_mode_steering = esw_vst_mode_is_steering(esw);
 858	u16 vport_num = vport->vport;
 859	int flags;
 860	int err;
 861
 862	err = esw_vport_setup_acl(esw, vport);
 863	if (err)
 864		return err;
 865
 866	if (mlx5_esw_is_manager_vport(esw, vport_num))
 867		return 0;
 868
 869	err = mlx5_esw_vport_caps_get(esw, vport);
 870	if (err)
 871		goto err_caps;
 872
 873	mlx5_modify_vport_admin_state(esw->dev,
 874				      MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 875				      vport_num, 1,
 876				      vport->info.link_state);
 877
 878	mlx5_query_nic_vport_mac_address(esw->dev, vport_num, true,
 879					 vport->info.mac);
 880	mlx5_query_nic_vport_node_guid(esw->dev, vport_num, true,
 881				       &vport->info.node_guid);
 882
 883	flags = (vport->info.vlan || vport->info.qos) ?
 884		SET_VLAN_STRIP | SET_VLAN_INSERT : 0;
 885	if (esw->mode == MLX5_ESWITCH_OFFLOADS || !vst_mode_steering)
 886		modify_esw_vport_cvlan(esw->dev, vport_num, vport->info.vlan,
 887				       vport->info.qos, flags);
 888
 889	return 0;
 890
 891err_caps:
 892	esw_vport_cleanup_acl(esw, vport);
 893	return err;
 894}
 895
 896/* Don't cleanup vport->info, it's needed to restore vport configuration */
 897static void esw_vport_cleanup(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 898{
 899	u16 vport_num = vport->vport;
 900
 901	if (!mlx5_esw_is_manager_vport(esw, vport_num))
 902		mlx5_modify_vport_admin_state(esw->dev,
 903					      MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 904					      vport_num, 1,
 905					      MLX5_VPORT_ADMIN_STATE_DOWN);
 906
 907	mlx5_esw_qos_vport_disable(vport);
 908	esw_vport_cleanup_acl(esw, vport);
 909}
 910
 911int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
 912			  enum mlx5_eswitch_vport_event enabled_events)
 913{
 914	u16 vport_num = vport->vport;
 915	int ret;
 916
 917	mutex_lock(&esw->state_lock);
 918	WARN_ON(vport->enabled);
 919
 920	esw_debug(esw->dev, "Enabling VPORT(%d)\n", vport_num);
 921
 922	ret = esw_vport_setup(esw, vport);
 923	if (ret)
 924		goto done;
 925
 926	/* Sync with current vport context */
 927	vport->enabled_events = enabled_events;
 928	vport->enabled = true;
 929	if (vport->vport != MLX5_VPORT_PF &&
 930	    (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled))
 931		esw->enabled_ipsec_vf_count++;
 932
 933	/* Esw manager is trusted by default. Host PF (vport 0) is trusted as well
 934	 * in smartNIC as it's a vport group manager.
 935	 */
 936	if (mlx5_esw_is_manager_vport(esw, vport_num) ||
 937	    (!vport_num && mlx5_core_is_ecpf(esw->dev)))
 938		vport->info.trusted = true;
 939
 940	if (!mlx5_esw_is_manager_vport(esw, vport_num) &&
 941	    MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
 942		ret = mlx5_esw_vport_vhca_id_map(esw, vport);
 943		if (ret)
 944			goto err_vhca_mapping;
 945	}
 946
 947	esw_vport_change_handle_locked(vport);
 948
 949	esw->enabled_vports++;
 950	esw_debug(esw->dev, "Enabled VPORT(%d)\n", vport_num);
 951done:
 952	mutex_unlock(&esw->state_lock);
 953	return ret;
 954
 955err_vhca_mapping:
 956	esw_vport_cleanup(esw, vport);
 957	mutex_unlock(&esw->state_lock);
 958	return ret;
 959}
 960
 961void mlx5_esw_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 962{
 963	u16 vport_num = vport->vport;
 964
 965	mutex_lock(&esw->state_lock);
 966
 967	if (!vport->enabled)
 968		goto done;
 969
 970	esw_debug(esw->dev, "Disabling vport(%d)\n", vport_num);
 971	/* Mark this vport as disabled to discard new events */
 972	vport->enabled = false;
 973
 974	/* Disable events from this vport */
 975	if (MLX5_CAP_GEN(esw->dev, log_max_l2_table))
 976		arm_vport_context_events_cmd(esw->dev, vport_num, 0);
 977
 978	if (!mlx5_esw_is_manager_vport(esw, vport_num) &&
 979	    MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
 980		mlx5_esw_vport_vhca_id_unmap(esw, vport);
 981
 982	if (vport->vport != MLX5_VPORT_PF &&
 983	    (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled))
 984		esw->enabled_ipsec_vf_count--;
 985
 986	/* We don't assume VFs will cleanup after themselves.
 987	 * Calling vport change handler while vport is disabled will cleanup
 988	 * the vport resources.
 989	 */
 990	esw_vport_change_handle_locked(vport);
 991	vport->enabled_events = 0;
 992	esw_apply_vport_rx_mode(esw, vport, false, false);
 993	esw_vport_cleanup(esw, vport);
 994	esw->enabled_vports--;
 995
 996done:
 997	mutex_unlock(&esw->state_lock);
 998}
 999
1000static int eswitch_vport_event(struct notifier_block *nb,
1001			       unsigned long type, void *data)
1002{
1003	struct mlx5_eswitch *esw = mlx5_nb_cof(nb, struct mlx5_eswitch, nb);
1004	struct mlx5_eqe *eqe = data;
1005	struct mlx5_vport *vport;
1006	u16 vport_num;
1007
1008	vport_num = be16_to_cpu(eqe->data.vport_change.vport_num);
1009	vport = mlx5_eswitch_get_vport(esw, vport_num);
1010	if (!IS_ERR(vport))
1011		queue_work(esw->work_queue, &vport->vport_change_handler);
1012	return NOTIFY_OK;
1013}
1014
1015/**
1016 * mlx5_esw_query_functions - Returns raw output about functions state
1017 * @dev:	Pointer to device to query
1018 *
1019 * mlx5_esw_query_functions() allocates and returns functions changed
1020 * raw output memory pointer from device on success. Otherwise returns ERR_PTR.
1021 * Caller must free the memory using kvfree() when valid pointer is returned.
1022 */
1023const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev)
1024{
1025	int outlen = MLX5_ST_SZ_BYTES(query_esw_functions_out);
1026	u32 in[MLX5_ST_SZ_DW(query_esw_functions_in)] = {};
1027	u32 *out;
1028	int err;
1029
1030	out = kvzalloc(outlen, GFP_KERNEL);
1031	if (!out)
1032		return ERR_PTR(-ENOMEM);
1033
1034	MLX5_SET(query_esw_functions_in, in, opcode,
1035		 MLX5_CMD_OP_QUERY_ESW_FUNCTIONS);
1036
1037	err = mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
1038	if (!err)
1039		return out;
1040
1041	kvfree(out);
1042	return ERR_PTR(err);
1043}
1044
1045static int mlx5_esw_host_functions_enabled_query(struct mlx5_eswitch *esw)
1046{
1047	const u32 *query_host_out;
1048
1049	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
1050		return 0;
1051
1052	query_host_out = mlx5_esw_query_functions(esw->dev);
1053	if (IS_ERR(query_host_out))
1054		return PTR_ERR(query_host_out);
1055
1056	esw->esw_funcs.host_funcs_disabled =
1057		MLX5_GET(query_esw_functions_out, query_host_out,
1058			 host_params_context.host_pf_not_exist);
1059
1060	kvfree(query_host_out);
1061	return 0;
1062}
1063
1064static void mlx5_eswitch_event_handler_register(struct mlx5_eswitch *esw)
1065{
1066	if (esw->mode == MLX5_ESWITCH_OFFLOADS && mlx5_eswitch_is_funcs_handler(esw->dev)) {
1067		MLX5_NB_INIT(&esw->esw_funcs.nb, mlx5_esw_funcs_changed_handler,
1068			     ESW_FUNCTIONS_CHANGED);
1069		mlx5_eq_notifier_register(esw->dev, &esw->esw_funcs.nb);
1070	}
1071}
1072
1073static void mlx5_eswitch_event_handler_unregister(struct mlx5_eswitch *esw)
1074{
1075	if (esw->mode == MLX5_ESWITCH_OFFLOADS &&
1076	    mlx5_eswitch_is_funcs_handler(esw->dev)) {
1077		mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb);
1078		atomic_inc(&esw->esw_funcs.generation);
1079	}
1080}
1081
1082static void mlx5_eswitch_clear_vf_vports_info(struct mlx5_eswitch *esw)
1083{
1084	struct mlx5_vport *vport;
1085	unsigned long i;
1086
1087	mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
1088		mlx5_esw_qos_vport_qos_free(vport);
1089		memset(&vport->info, 0, sizeof(vport->info));
1090		vport->info.link_state = MLX5_VPORT_ADMIN_STATE_AUTO;
1091	}
1092}
1093
1094static void mlx5_eswitch_clear_ec_vf_vports_info(struct mlx5_eswitch *esw)
1095{
1096	struct mlx5_vport *vport;
1097	unsigned long i;
1098
1099	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) {
1100		mlx5_esw_qos_vport_qos_free(vport);
1101		memset(&vport->info, 0, sizeof(vport->info));
1102		vport->info.link_state = MLX5_VPORT_ADMIN_STATE_AUTO;
1103	}
1104}
1105
1106static int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
1107				   enum mlx5_eswitch_vport_event enabled_events)
1108{
1109	int err;
1110
1111	err = mlx5_esw_vport_enable(esw, vport, enabled_events);
1112	if (err)
1113		return err;
1114
1115	err = mlx5_esw_offloads_load_rep(esw, vport);
1116	if (err)
1117		goto err_rep;
1118
1119	return err;
1120
1121err_rep:
1122	mlx5_esw_vport_disable(esw, vport);
1123	return err;
1124}
1125
1126static void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
1127{
1128	mlx5_esw_offloads_unload_rep(esw, vport);
1129	mlx5_esw_vport_disable(esw, vport);
1130}
1131
1132static int mlx5_eswitch_load_pf_vf_vport(struct mlx5_eswitch *esw, u16 vport_num,
1133					 enum mlx5_eswitch_vport_event enabled_events)
1134{
1135	struct mlx5_vport *vport;
1136	int err;
1137
1138	vport = mlx5_eswitch_get_vport(esw, vport_num);
1139	if (IS_ERR(vport))
1140		return PTR_ERR(vport);
1141
1142	err = mlx5_esw_offloads_init_pf_vf_rep(esw, vport);
1143	if (err)
1144		return err;
1145
1146	err = mlx5_eswitch_load_vport(esw, vport, enabled_events);
1147	if (err)
1148		goto err_load;
1149	return 0;
1150
1151err_load:
1152	mlx5_esw_offloads_cleanup_pf_vf_rep(esw, vport);
1153	return err;
1154}
1155
1156static void mlx5_eswitch_unload_pf_vf_vport(struct mlx5_eswitch *esw, u16 vport_num)
1157{
1158	struct mlx5_vport *vport;
1159
1160	vport = mlx5_eswitch_get_vport(esw, vport_num);
1161	if (IS_ERR(vport))
1162		return;
1163
1164	mlx5_eswitch_unload_vport(esw, vport);
1165	mlx5_esw_offloads_cleanup_pf_vf_rep(esw, vport);
1166}
1167
1168int mlx5_eswitch_load_sf_vport(struct mlx5_eswitch *esw, u16 vport_num,
1169			       enum mlx5_eswitch_vport_event enabled_events,
1170			       struct mlx5_devlink_port *dl_port, u32 controller, u32 sfnum)
1171{
1172	struct mlx5_vport *vport;
1173	int err;
1174
1175	vport = mlx5_eswitch_get_vport(esw, vport_num);
1176	if (IS_ERR(vport))
1177		return PTR_ERR(vport);
1178
1179	err = mlx5_esw_offloads_init_sf_rep(esw, vport, dl_port, controller, sfnum);
1180	if (err)
1181		return err;
1182
1183	err = mlx5_eswitch_load_vport(esw, vport, enabled_events);
1184	if (err)
1185		goto err_load;
1186
1187	return 0;
1188
1189err_load:
1190	mlx5_esw_offloads_cleanup_sf_rep(esw, vport);
1191	return err;
1192}
1193
1194void mlx5_eswitch_unload_sf_vport(struct mlx5_eswitch *esw, u16 vport_num)
1195{
1196	struct mlx5_vport *vport;
1197
1198	vport = mlx5_eswitch_get_vport(esw, vport_num);
1199	if (IS_ERR(vport))
1200		return;
1201
1202	mlx5_eswitch_unload_vport(esw, vport);
1203	mlx5_esw_offloads_cleanup_sf_rep(esw, vport);
1204}
1205
1206void mlx5_eswitch_unload_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs)
1207{
1208	struct mlx5_vport *vport;
1209	unsigned long i;
1210
1211	mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) {
1212		/* Adjacent VFs are unloaded separately */
1213		if (!vport->enabled || vport->adjacent)
1214			continue;
1215		mlx5_eswitch_unload_pf_vf_vport(esw, vport->vport);
1216	}
1217}
1218
1219static void mlx5_eswitch_unload_ec_vf_vports(struct mlx5_eswitch *esw,
1220					     u16 num_ec_vfs)
1221{
1222	struct mlx5_vport *vport;
1223	unsigned long i;
1224
1225	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, num_ec_vfs) {
1226		if (!vport->enabled)
1227			continue;
1228		mlx5_eswitch_unload_pf_vf_vport(esw, vport->vport);
1229	}
1230}
1231
1232static void mlx5_eswitch_unload_adj_vf_vports(struct mlx5_eswitch *esw)
1233{
1234	struct mlx5_vport *vport;
1235	unsigned long i;
1236
1237	mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
1238		if (!vport->enabled || !vport->adjacent)
1239			continue;
1240		mlx5_eswitch_unload_pf_vf_vport(esw, vport->vport);
1241	}
1242}
1243
1244static int
1245mlx5_eswitch_load_adj_vf_vports(struct mlx5_eswitch *esw,
1246				enum mlx5_eswitch_vport_event enabled_events)
1247{
1248	struct mlx5_vport *vport;
1249	unsigned long i;
1250	int err;
1251
1252	mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) {
1253		if (!vport->adjacent)
1254			continue;
1255		err = mlx5_eswitch_load_pf_vf_vport(esw, vport->vport,
1256						    enabled_events);
1257		if (err)
1258			goto unload_adj_vf_vport;
1259	}
1260
1261	return 0;
1262
1263unload_adj_vf_vport:
1264	mlx5_eswitch_unload_adj_vf_vports(esw);
1265	return err;
1266}
1267
1268int mlx5_eswitch_load_vf_vports(struct mlx5_eswitch *esw, u16 num_vfs,
1269				enum mlx5_eswitch_vport_event enabled_events)
1270{
1271	struct mlx5_vport *vport;
1272	unsigned long i;
1273	int err;
1274
1275	mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) {
1276		err = mlx5_eswitch_load_pf_vf_vport(esw, vport->vport, enabled_events);
1277		if (err)
1278			goto vf_err;
1279	}
1280
1281	return 0;
1282
1283vf_err:
1284	mlx5_eswitch_unload_vf_vports(esw, num_vfs);
1285	return err;
1286}
1287
1288static int mlx5_eswitch_load_ec_vf_vports(struct mlx5_eswitch *esw, u16 num_ec_vfs,
1289					  enum mlx5_eswitch_vport_event enabled_events)
1290{
1291	struct mlx5_vport *vport;
1292	unsigned long i;
1293	int err;
1294
1295	mlx5_esw_for_each_ec_vf_vport(esw, i, vport, num_ec_vfs) {
1296		err = mlx5_eswitch_load_pf_vf_vport(esw, vport->vport, enabled_events);
1297		if (err)
1298			goto vf_err;
1299	}
1300
1301	return 0;
1302
1303vf_err:
1304	mlx5_eswitch_unload_ec_vf_vports(esw, num_ec_vfs);
1305	return err;
1306}
1307
1308static int host_pf_enable_hca(struct mlx5_core_dev *dev)
1309{
1310	if (!mlx5_core_is_ecpf(dev))
1311		return 0;
1312
1313	/* Once vport and representor are ready, take out the external host PF
1314	 * out of initializing state. Enabling HCA clears the iser->initializing
1315	 * bit and host PF driver loading can progress.
1316	 */
1317	return mlx5_cmd_host_pf_enable_hca(dev);
1318}
1319
1320static void host_pf_disable_hca(struct mlx5_core_dev *dev)
1321{
1322	if (!mlx5_core_is_ecpf(dev))
1323		return;
1324
1325	mlx5_cmd_host_pf_disable_hca(dev);
1326}
1327
1328/* mlx5_eswitch_enable_pf_vf_vports() enables vports of PF, ECPF and VFs
1329 * whichever are present on the eswitch.
1330 */
1331int
1332mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw,
1333				 enum mlx5_eswitch_vport_event enabled_events)
1334{
1335	bool pf_needed;
1336	int ret;
1337
1338	pf_needed = mlx5_core_is_ecpf_esw_manager(esw->dev) ||
1339		    esw->mode == MLX5_ESWITCH_LEGACY;
1340
1341	/* Enable PF vport */
1342	if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev)) {
1343		ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_PF,
1344						    enabled_events);
1345		if (ret)
1346			return ret;
1347	}
1348
1349	if (mlx5_esw_host_functions_enabled(esw->dev)) {
1350		/* Enable external host PF HCA */
1351		ret = host_pf_enable_hca(esw->dev);
1352		if (ret)
1353			goto pf_hca_err;
1354	}
1355
1356	/* Enable ECPF vport */
1357	if (mlx5_ecpf_vport_exists(esw->dev)) {
1358		ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_ECPF, enabled_events);
1359		if (ret)
1360			goto ecpf_err;
1361	}
1362
1363	/* Enable ECVF vports */
1364	if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1365		ret = mlx5_eswitch_load_ec_vf_vports(esw,
1366						     esw->esw_funcs.num_ec_vfs,
1367						     enabled_events);
1368		if (ret)
1369			goto ec_vf_err;
1370	}
1371
1372	/* Enable VF vports */
1373	ret = mlx5_eswitch_load_vf_vports(esw, esw->esw_funcs.num_vfs,
1374					  enabled_events);
1375	if (ret)
1376		goto vf_err;
1377
1378	/* Enable adjacent VF vports */
1379	ret = mlx5_eswitch_load_adj_vf_vports(esw, enabled_events);
1380	if (ret)
1381		goto unload_vf_vports;
1382
1383	return 0;
1384
1385unload_vf_vports:
1386	mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
1387vf_err:
1388	if (mlx5_core_ec_sriov_enabled(esw->dev))
1389		mlx5_eswitch_unload_ec_vf_vports(esw, esw->esw_funcs.num_ec_vfs);
1390ec_vf_err:
1391	if (mlx5_ecpf_vport_exists(esw->dev))
1392		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_ECPF);
1393ecpf_err:
1394	if (mlx5_esw_host_functions_enabled(esw->dev))
1395		host_pf_disable_hca(esw->dev);
1396pf_hca_err:
1397	if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev))
1398		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF);
1399	return ret;
1400}
1401
1402/* mlx5_eswitch_disable_pf_vf_vports() disables vports of PF, ECPF and VFs
1403 * whichever are previously enabled on the eswitch.
1404 */
1405void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw)
1406{
1407	mlx5_eswitch_unload_adj_vf_vports(esw);
1408
1409	mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
1410
1411	if (mlx5_core_ec_sriov_enabled(esw->dev))
1412		mlx5_eswitch_unload_ec_vf_vports(esw,
1413						 esw->esw_funcs.num_ec_vfs);
1414
1415	if (mlx5_ecpf_vport_exists(esw->dev)) {
1416		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_ECPF);
1417	}
1418
1419	if (mlx5_esw_host_functions_enabled(esw->dev))
1420		host_pf_disable_hca(esw->dev);
1421
1422	if ((mlx5_core_is_ecpf_esw_manager(esw->dev) ||
1423	     esw->mode == MLX5_ESWITCH_LEGACY) &&
1424	    mlx5_esw_host_functions_enabled(esw->dev))
1425		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF);
1426}
1427
1428static void mlx5_eswitch_get_devlink_param(struct mlx5_eswitch *esw)
1429{
1430	struct devlink *devlink = priv_to_devlink(esw->dev);
1431	union devlink_param_value val;
1432	int err;
1433
1434	err = devl_param_driverinit_value_get(devlink,
1435					      MLX5_DEVLINK_PARAM_ID_ESW_LARGE_GROUP_NUM,
1436					      &val);
1437	if (!err) {
1438		esw->params.large_group_num = val.vu32;
1439	} else {
1440		esw_warn(esw->dev,
1441			 "Devlink can't get param fdb_large_groups, uses default (%d).\n",
1442			 ESW_OFFLOADS_DEFAULT_NUM_GROUPS);
1443		esw->params.large_group_num = ESW_OFFLOADS_DEFAULT_NUM_GROUPS;
1444	}
1445}
1446
1447static void
1448mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs)
1449{
1450	const u32 *out;
1451
1452	if (num_vfs < 0)
1453		return;
1454
1455	if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1456		esw->esw_funcs.num_vfs = num_vfs;
1457		return;
1458	}
1459
1460	out = mlx5_esw_query_functions(esw->dev);
1461	if (IS_ERR(out))
1462		return;
1463
1464	esw->esw_funcs.num_vfs = MLX5_GET(query_esw_functions_out, out,
1465					  host_params_context.host_num_of_vfs);
1466	if (mlx5_core_ec_sriov_enabled(esw->dev))
1467		esw->esw_funcs.num_ec_vfs = num_vfs;
1468
1469	kvfree(out);
1470}
1471
1472static void mlx5_esw_mode_change_notify(struct mlx5_eswitch *esw, u16 mode)
1473{
1474	struct mlx5_esw_event_info info = {};
1475
1476	info.new_mode = mode;
1477
1478	blocking_notifier_call_chain(&esw->dev->priv.esw_n_head, 0, &info);
1479}
1480
1481static int mlx5_esw_egress_acls_init(struct mlx5_core_dev *dev)
1482{
1483	struct mlx5_flow_steering *steering = dev->priv.steering;
1484	int total_vports = mlx5_eswitch_get_total_vports(dev);
1485	int err;
1486	int i;
1487
1488	for (i = 0; i < total_vports; i++) {
1489		err = mlx5_fs_vport_egress_acl_ns_add(steering, i);
1490		if (err)
1491			goto acl_ns_remove;
1492	}
1493	return 0;
1494
1495acl_ns_remove:
1496	while (i--)
1497		mlx5_fs_vport_egress_acl_ns_remove(steering, i);
1498	return err;
1499}
1500
1501static void mlx5_esw_egress_acls_cleanup(struct mlx5_core_dev *dev)
1502{
1503	struct mlx5_flow_steering *steering = dev->priv.steering;
1504	int total_vports = mlx5_eswitch_get_total_vports(dev);
1505	int i;
1506
1507	for (i = total_vports - 1; i >= 0; i--)
1508		mlx5_fs_vport_egress_acl_ns_remove(steering, i);
1509}
1510
1511static int mlx5_esw_ingress_acls_init(struct mlx5_core_dev *dev)
1512{
1513	struct mlx5_flow_steering *steering = dev->priv.steering;
1514	int total_vports = mlx5_eswitch_get_total_vports(dev);
1515	int err;
1516	int i;
1517
1518	for (i = 0; i < total_vports; i++) {
1519		err = mlx5_fs_vport_ingress_acl_ns_add(steering, i);
1520		if (err)
1521			goto acl_ns_remove;
1522	}
1523	return 0;
1524
1525acl_ns_remove:
1526	while (i--)
1527		mlx5_fs_vport_ingress_acl_ns_remove(steering, i);
1528	return err;
1529}
1530
1531static void mlx5_esw_ingress_acls_cleanup(struct mlx5_core_dev *dev)
1532{
1533	struct mlx5_flow_steering *steering = dev->priv.steering;
1534	int total_vports = mlx5_eswitch_get_total_vports(dev);
1535	int i;
1536
1537	for (i = total_vports - 1; i >= 0; i--)
1538		mlx5_fs_vport_ingress_acl_ns_remove(steering, i);
1539}
1540
1541static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw)
1542{
1543	struct mlx5_core_dev *dev = esw->dev;
1544	int err;
1545
1546	if (esw->flags & MLX5_ESWITCH_VPORT_ACL_NS_CREATED)
1547		return 0;
1548
1549	if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) {
1550		err = mlx5_esw_egress_acls_init(dev);
1551		if (err)
1552			return err;
1553	} else {
1554		esw_warn(dev, "egress ACL is not supported by FW\n");
1555	}
1556
1557	if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) {
1558		err = mlx5_esw_ingress_acls_init(dev);
1559		if (err)
1560			goto err;
1561	} else {
1562		esw_warn(dev, "ingress ACL is not supported by FW\n");
1563	}
1564	esw->flags |= MLX5_ESWITCH_VPORT_ACL_NS_CREATED;
1565	return 0;
1566
1567err:
1568	if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support))
1569		mlx5_esw_egress_acls_cleanup(dev);
1570	return err;
1571}
1572
1573static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw)
1574{
1575	struct mlx5_core_dev *dev = esw->dev;
1576
1577	esw->flags &= ~MLX5_ESWITCH_VPORT_ACL_NS_CREATED;
1578	if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support))
1579		mlx5_esw_ingress_acls_cleanup(dev);
1580	if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support))
1581		mlx5_esw_egress_acls_cleanup(dev);
1582}
1583
1584/**
1585 * mlx5_eswitch_enable_locked - Enable eswitch
1586 * @esw:	Pointer to eswitch
1587 * @num_vfs:	Enable eswitch for given number of VFs. This is optional.
1588 *		Valid value are 0, > 0 and MLX5_ESWITCH_IGNORE_NUM_VFS.
1589 *		Caller should pass num_vfs > 0 when enabling eswitch for
1590 *		vf vports. Caller should pass num_vfs = 0, when eswitch
1591 *		is enabled without sriov VFs or when caller
1592 *		is unaware of the sriov state of the host PF on ECPF based
1593 *		eswitch. Caller should pass < 0 when num_vfs should be
1594 *		completely ignored. This is typically the case when eswitch
1595 *		is enabled without sriov regardless of PF/ECPF system.
1596 * mlx5_eswitch_enable_locked() Enables eswitch in either legacy or offloads
1597 * mode. If num_vfs >=0 is provided, it setup VF related eswitch vports.
1598 * It returns 0 on success or error code on failure.
1599 */
1600int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs)
1601{
1602	int err;
1603
1604	devl_assert_locked(priv_to_devlink(esw->dev));
1605
1606	if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ft_support)) {
1607		esw_warn(esw->dev, "FDB is not supported, aborting ...\n");
1608		return -EOPNOTSUPP;
1609	}
1610
1611	mlx5_eswitch_get_devlink_param(esw);
1612
1613	err = mlx5_esw_acls_ns_init(esw);
1614	if (err)
1615		return err;
1616
1617	mlx5_eswitch_update_num_of_vfs(esw, num_vfs);
1618
1619	MLX5_NB_INIT(&esw->nb, eswitch_vport_event, NIC_VPORT_CHANGE);
1620	mlx5_eq_notifier_register(esw->dev, &esw->nb);
1621
1622	err = mlx5_esw_qos_init(esw);
1623	if (err)
1624		goto err_esw_init;
1625
1626	if (esw->mode == MLX5_ESWITCH_LEGACY) {
1627		err = esw_legacy_enable(esw);
1628	} else {
1629		err = esw_offloads_enable(esw);
1630	}
1631
1632	if (err)
1633		goto err_esw_init;
1634
1635	esw->fdb_table.flags |= MLX5_ESW_FDB_CREATED;
1636
1637	mlx5_eswitch_event_handler_register(esw);
1638
1639	esw_info(esw->dev, "Enable: mode(%s), nvfs(%d), necvfs(%d), active vports(%d)\n",
1640		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
1641		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
1642
1643	mlx5_esw_mode_change_notify(esw, esw->mode);
1644
1645	return 0;
1646
1647err_esw_init:
1648	mlx5_eq_notifier_unregister(esw->dev, &esw->nb);
1649	mlx5_esw_acls_ns_cleanup(esw);
1650	return err;
1651}
1652
1653/**
1654 * mlx5_eswitch_enable - Enable eswitch
1655 * @esw:	Pointer to eswitch
1656 * @num_vfs:	Enable eswitch switch for given number of VFs.
1657 *		Caller must pass num_vfs > 0 when enabling eswitch for
1658 *		vf vports.
1659 * mlx5_eswitch_enable() returns 0 on success or error code on failure.
1660 */
1661int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
1662{
1663	bool toggle_lag;
1664	int ret = 0;
1665
1666	if (!mlx5_esw_allowed(esw))
1667		return 0;
1668
1669	devl_assert_locked(priv_to_devlink(esw->dev));
1670
1671	toggle_lag = !mlx5_esw_is_fdb_created(esw);
1672
1673	if (toggle_lag)
1674		mlx5_lag_disable_change(esw->dev);
1675
1676	if (!mlx5_esw_is_fdb_created(esw)) {
1677		ret = mlx5_eswitch_enable_locked(esw, num_vfs);
1678	} else {
1679		enum mlx5_eswitch_vport_event vport_events;
1680
1681		vport_events = (esw->mode == MLX5_ESWITCH_LEGACY) ?
1682					MLX5_LEGACY_SRIOV_VPORT_EVENTS : MLX5_VPORT_UC_ADDR_CHANGE;
1683		/* If this is the ECPF the number of host VFs is managed via the
1684		 * eswitch function change event handler, and any num_vfs provided
1685		 * here are intended to be EC VFs.
1686		 */
1687		if (!mlx5_core_is_ecpf(esw->dev)) {
1688			ret = mlx5_eswitch_load_vf_vports(esw, num_vfs, vport_events);
1689			if (!ret)
1690				esw->esw_funcs.num_vfs = num_vfs;
1691		} else if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1692			ret = mlx5_eswitch_load_ec_vf_vports(esw, num_vfs, vport_events);
1693			if (!ret)
1694				esw->esw_funcs.num_ec_vfs = num_vfs;
1695		}
1696	}
1697
1698	if (toggle_lag)
1699		mlx5_lag_enable_change(esw->dev);
1700
1701	return ret;
1702}
1703
1704/* When disabling sriov, free driver level resources. */
1705void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
1706{
1707	if (!mlx5_esw_allowed(esw))
1708		return;
1709
1710	devl_assert_locked(priv_to_devlink(esw->dev));
1711	/* If driver is unloaded, this function is called twice by remove_one()
1712	 * and mlx5_unload(). Prevent the second call.
1713	 */
1714	if (!esw->esw_funcs.num_vfs && !esw->esw_funcs.num_ec_vfs && !clear_vf)
1715		return;
1716
1717	esw_info(esw->dev, "Unload vfs: mode(%s), nvfs(%d), necvfs(%d), active vports(%d)\n",
1718		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
1719		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
1720
1721	if (!mlx5_core_is_ecpf(esw->dev)) {
1722		mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
1723		if (clear_vf)
1724			mlx5_eswitch_clear_vf_vports_info(esw);
1725	} else if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1726		mlx5_eswitch_unload_ec_vf_vports(esw, esw->esw_funcs.num_ec_vfs);
1727		if (clear_vf)
1728			mlx5_eswitch_clear_ec_vf_vports_info(esw);
1729	}
1730
1731	if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
1732		struct devlink *devlink = priv_to_devlink(esw->dev);
1733
1734		devl_rate_nodes_destroy(devlink);
1735	}
1736	/* Destroy legacy fdb when disabling sriov in legacy mode. */
1737	if (esw->mode == MLX5_ESWITCH_LEGACY)
1738		mlx5_eswitch_disable_locked(esw);
1739
1740	if (!mlx5_core_is_ecpf(esw->dev))
1741		esw->esw_funcs.num_vfs = 0;
1742	else
1743		esw->esw_funcs.num_ec_vfs = 0;
1744}
1745
1746/* Free resources for corresponding eswitch mode. It is called by devlink
1747 * when changing eswitch mode or modprobe when unloading driver.
1748 */
1749void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw)
1750{
1751	struct devlink *devlink = priv_to_devlink(esw->dev);
1752
1753	/* Notify eswitch users that it is exiting from current mode.
1754	 * So that it can do necessary cleanup before the eswitch is disabled.
1755	 */
1756	mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_LEGACY);
1757
1758	mlx5_eq_notifier_unregister(esw->dev, &esw->nb);
1759	mlx5_eswitch_event_handler_unregister(esw);
1760
1761	esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), necvfs(%d), active vports(%d)\n",
1762		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
1763		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
1764
1765	if (esw->fdb_table.flags & MLX5_ESW_FDB_CREATED) {
1766		esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED;
1767		if (esw->mode == MLX5_ESWITCH_OFFLOADS)
1768			esw_offloads_disable(esw);
1769		else if (esw->mode == MLX5_ESWITCH_LEGACY)
1770			esw_legacy_disable(esw);
1771		mlx5_esw_acls_ns_cleanup(esw);
1772	}
1773
1774	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
1775		devl_rate_nodes_destroy(devlink);
1776}
1777
1778void mlx5_eswitch_disable(struct mlx5_eswitch *esw)
1779{
1780	if (!mlx5_esw_allowed(esw))
1781		return;
1782
1783	devl_assert_locked(priv_to_devlink(esw->dev));
1784	mlx5_lag_disable_change(esw->dev);
1785	mlx5_eswitch_disable_locked(esw);
1786	esw->mode = MLX5_ESWITCH_LEGACY;
1787	mlx5_lag_enable_change(esw->dev);
1788}
1789
1790static int mlx5_query_hca_cap_host_pf(struct mlx5_core_dev *dev, void *out)
1791{
1792	u16 opmod = (MLX5_CAP_GENERAL << 1) | (HCA_CAP_OPMOD_GET_MAX & 0x01);
1793	u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)] = {};
1794
1795	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
1796	MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
1797	MLX5_SET(query_hca_cap_in, in, function_id, MLX5_VPORT_PF);
1798	MLX5_SET(query_hca_cap_in, in, other_function, true);
1799	return mlx5_cmd_exec_inout(dev, query_hca_cap, in, out);
1800}
1801
1802int mlx5_esw_sf_max_hpf_functions(struct mlx5_core_dev *dev, u16 *max_sfs, u16 *sf_base_id)
1803
1804{
1805	int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
1806	void *query_ctx;
1807	void *hca_caps;
1808	int err;
1809
1810	if (!mlx5_core_is_ecpf(dev) ||
1811	    !mlx5_esw_host_functions_enabled(dev)) {
1812		*max_sfs = 0;
1813		return 0;
1814	}
1815
1816	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
1817	if (!query_ctx)
1818		return -ENOMEM;
1819
1820	err = mlx5_query_hca_cap_host_pf(dev, query_ctx);
1821	if (err)
1822		goto out_free;
1823
1824	hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
1825	*max_sfs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_sf);
1826	*sf_base_id = MLX5_GET(cmd_hca_cap, hca_caps, sf_base_id);
1827
1828out_free:
1829	kfree(query_ctx);
1830	return err;
1831}
1832
1833int mlx5_esw_vport_alloc(struct mlx5_eswitch *esw, int index, u16 vport_num)
1834{
1835	struct mlx5_vport *vport;
1836	int err;
1837
1838	vport = kzalloc(sizeof(*vport), GFP_KERNEL);
1839	if (!vport)
1840		return -ENOMEM;
1841
1842	vport->dev = esw->dev;
1843	vport->vport = vport_num;
1844	vport->index = index;
1845	vport->info.link_state = MLX5_VPORT_ADMIN_STATE_AUTO;
1846	vport->vhca_id = MLX5_VHCA_ID_INVALID;
1847	INIT_WORK(&vport->vport_change_handler, esw_vport_change_handler);
1848	err = xa_insert(&esw->vports, vport_num, vport, GFP_KERNEL);
1849	if (err)
1850		goto insert_err;
1851
1852	esw->total_vports++;
1853	return 0;
1854
1855insert_err:
1856	kfree(vport);
1857	return err;
1858}
1859
1860void mlx5_esw_vport_free(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
1861{
1862	esw->total_vports--;
1863	xa_erase(&esw->vports, vport->vport);
1864	kfree(vport);
1865}
1866
1867static void mlx5_esw_vports_cleanup(struct mlx5_eswitch *esw)
1868{
1869	struct mlx5_vport *vport;
1870	unsigned long i;
1871
1872	mlx5_esw_for_each_vport(esw, i, vport)
1873		mlx5_esw_vport_free(esw, vport);
1874	xa_destroy(&esw->vports);
1875}
1876
1877static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
1878{
1879	struct mlx5_core_dev *dev = esw->dev;
1880	u16 max_host_pf_sfs;
1881	u16 base_sf_num;
1882	int idx = 0;
1883	int err;
1884	int i;
1885
1886	xa_init(&esw->vports);
1887
1888	if (mlx5_esw_host_functions_enabled(dev)) {
1889		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_PF);
1890		if (err)
1891			goto err;
1892		if (esw->first_host_vport == MLX5_VPORT_PF)
1893			xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN);
1894		idx++;
1895		for (i = 0; i < mlx5_core_max_vfs(dev); i++) {
1896			err = mlx5_esw_vport_alloc(esw, idx, idx);
1897			if (err)
1898				goto err;
1899			xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_VF);
1900			xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN);
1901			idx++;
1902		}
1903	}
1904
1905	base_sf_num = mlx5_sf_start_function_id(dev);
1906	for (i = 0; i < mlx5_sf_max_functions(dev); i++) {
1907		err = mlx5_esw_vport_alloc(esw, idx, base_sf_num + i);
1908		if (err)
1909			goto err;
1910		xa_set_mark(&esw->vports, base_sf_num + i, MLX5_ESW_VPT_SF);
1911		idx++;
1912	}
1913
1914	err = mlx5_esw_sf_max_hpf_functions(dev, &max_host_pf_sfs, &base_sf_num);
1915	if (err)
1916		goto err;
1917	for (i = 0; i < max_host_pf_sfs; i++) {
1918		err = mlx5_esw_vport_alloc(esw, idx, base_sf_num + i);
1919		if (err)
1920			goto err;
1921		xa_set_mark(&esw->vports, base_sf_num + i, MLX5_ESW_VPT_SF);
1922		idx++;
1923	}
1924
1925	if (mlx5_core_ec_sriov_enabled(esw->dev)) {
1926		int ec_vf_base_num = mlx5_core_ec_vf_vport_base(dev);
1927
1928		for (i = 0; i < mlx5_core_max_ec_vfs(esw->dev); i++) {
1929			err = mlx5_esw_vport_alloc(esw, idx, ec_vf_base_num + i);
1930			if (err)
1931				goto err;
1932			idx++;
1933		}
1934	}
1935
1936	if (mlx5_ecpf_vport_exists(dev) ||
1937	    mlx5_core_is_ecpf_esw_manager(dev)) {
1938		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_ECPF);
1939		if (err)
1940			goto err;
1941		idx++;
1942	}
1943	err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_UPLINK);
1944	if (err)
1945		goto err;
1946
1947	/* Adjacent vports or other dynamically create vports will use this */
1948	esw->last_vport_idx = ++idx;
1949	return 0;
1950
1951err:
1952	mlx5_esw_vports_cleanup(esw);
1953	return err;
1954}
1955
1956static int mlx5_devlink_esw_multiport_set(struct devlink *devlink, u32 id,
1957					  struct devlink_param_gset_ctx *ctx,
1958					  struct netlink_ext_ack *extack)
1959{
1960	struct mlx5_core_dev *dev = devlink_priv(devlink);
1961
1962	if (!MLX5_ESWITCH_MANAGER(dev))
1963		return -EOPNOTSUPP;
1964
1965	if (ctx->val.vbool)
1966		return mlx5_lag_mpesw_enable(dev);
1967
1968	mlx5_lag_mpesw_disable(dev);
1969	return 0;
1970}
1971
1972static int mlx5_devlink_esw_multiport_get(struct devlink *devlink, u32 id,
1973					  struct devlink_param_gset_ctx *ctx,
1974					  struct netlink_ext_ack *extack)
1975{
1976	struct mlx5_core_dev *dev = devlink_priv(devlink);
1977
1978	ctx->val.vbool = mlx5_lag_is_mpesw(dev);
1979	return 0;
1980}
1981
1982static const struct devlink_param mlx5_eswitch_params[] = {
1983	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_MULTIPORT,
1984			     "esw_multiport", DEVLINK_PARAM_TYPE_BOOL,
1985			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),
1986			     mlx5_devlink_esw_multiport_get,
1987			     mlx5_devlink_esw_multiport_set, NULL),
1988};
1989
1990int mlx5_eswitch_init(struct mlx5_core_dev *dev)
1991{
1992	struct mlx5_eswitch *esw;
1993	int err;
1994
1995	if (!MLX5_VPORT_MANAGER(dev) && !MLX5_ESWITCH_MANAGER(dev))
1996		return 0;
1997
1998	esw = kzalloc(sizeof(*esw), GFP_KERNEL);
1999	if (!esw)
2000		return -ENOMEM;
2001
2002	err = devl_params_register(priv_to_devlink(dev), mlx5_eswitch_params,
2003				   ARRAY_SIZE(mlx5_eswitch_params));
2004	if (err)
2005		goto free_esw;
2006
2007	esw->dev = dev;
2008	dev->priv.eswitch = esw;
2009	esw->manager_vport = mlx5_eswitch_manager_vport(dev);
2010	esw->first_host_vport = mlx5_eswitch_first_host_vport_num(dev);
2011
2012	esw->debugfs_root = debugfs_create_dir("esw", mlx5_debugfs_get_dev_root(dev));
2013	esw->work_queue = create_singlethread_workqueue("mlx5_esw_wq");
2014	if (!esw->work_queue) {
2015		err = -ENOMEM;
2016		goto abort;
2017	}
2018
2019	err = mlx5_esw_host_functions_enabled_query(esw);
2020	if (err)
2021		goto abort;
2022
2023	err = mlx5_esw_vports_init(esw);
2024	if (err)
2025		goto abort;
2026
2027	err = esw_offloads_init(esw);
2028	if (err)
2029		goto reps_err;
2030
2031	esw->mode = MLX5_ESWITCH_LEGACY;
2032	err = mlx5_esw_qos_init(esw);
2033	if (err)
2034		goto reps_err;
2035
2036	mutex_init(&esw->offloads.encap_tbl_lock);
2037	hash_init(esw->offloads.encap_tbl);
2038	mutex_init(&esw->offloads.decap_tbl_lock);
2039	hash_init(esw->offloads.decap_tbl);
2040	mlx5e_mod_hdr_tbl_init(&esw->offloads.mod_hdr);
2041	atomic64_set(&esw->offloads.num_flows, 0);
2042	ida_init(&esw->offloads.vport_metadata_ida);
2043	xa_init_flags(&esw->offloads.vhca_map, XA_FLAGS_ALLOC);
2044	mutex_init(&esw->state_lock);
2045	init_rwsem(&esw->mode_lock);
2046	refcount_set(&esw->qos.refcnt, 0);
2047
2048	esw->enabled_vports = 0;
2049	esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE;
2050	if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) &&
2051	    MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))
2052		esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC;
2053	else
2054		esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
2055
2056	esw_info(dev,
2057		 "Total vports %d, per vport: max uc(%d) max mc(%d)\n",
2058		 esw->total_vports,
2059		 MLX5_MAX_UC_PER_VPORT(dev),
2060		 MLX5_MAX_MC_PER_VPORT(dev));
2061	return 0;
2062
2063reps_err:
2064	mlx5_esw_vports_cleanup(esw);
2065	dev->priv.eswitch = NULL;
2066abort:
2067	if (esw->work_queue)
2068		destroy_workqueue(esw->work_queue);
2069	debugfs_remove_recursive(esw->debugfs_root);
2070	devl_params_unregister(priv_to_devlink(dev), mlx5_eswitch_params,
2071			       ARRAY_SIZE(mlx5_eswitch_params));
2072free_esw:
2073	kfree(esw);
2074	return err;
2075}
2076
2077void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
2078{
2079	if (!esw)
2080		return;
2081
2082	esw_info(esw->dev, "cleanup\n");
2083
2084	mlx5_esw_qos_cleanup(esw);
2085	destroy_workqueue(esw->work_queue);
2086	WARN_ON(refcount_read(&esw->qos.refcnt));
2087	mutex_destroy(&esw->state_lock);
2088	WARN_ON(!xa_empty(&esw->offloads.vhca_map));
2089	xa_destroy(&esw->offloads.vhca_map);
2090	ida_destroy(&esw->offloads.vport_metadata_ida);
2091	mlx5e_mod_hdr_tbl_destroy(&esw->offloads.mod_hdr);
2092	mutex_destroy(&esw->offloads.encap_tbl_lock);
2093	mutex_destroy(&esw->offloads.decap_tbl_lock);
2094	esw_offloads_cleanup(esw);
2095	esw->dev->priv.eswitch = NULL;
2096	mlx5_esw_vports_cleanup(esw);
2097	debugfs_remove_recursive(esw->debugfs_root);
2098	devl_params_unregister(priv_to_devlink(esw->dev), mlx5_eswitch_params,
2099			       ARRAY_SIZE(mlx5_eswitch_params));
2100	kfree(esw);
2101}
2102
2103/* Vport Administration */
2104static int
2105mlx5_esw_set_vport_mac_locked(struct mlx5_eswitch *esw,
2106			      struct mlx5_vport *evport, const u8 *mac)
2107{
2108	u16 vport_num = evport->vport;
2109	u64 node_guid;
2110	int err = 0;
2111
2112	if (is_multicast_ether_addr(mac))
2113		return -EINVAL;
2114
2115	if (evport->info.spoofchk && !is_valid_ether_addr(mac))
2116		mlx5_core_warn(esw->dev,
2117			       "Set invalid MAC while spoofchk is on, vport(%d)\n",
2118			       vport_num);
2119
2120	err = mlx5_modify_nic_vport_mac_address(esw->dev, vport_num, mac);
2121	if (err) {
2122		mlx5_core_warn(esw->dev,
2123			       "Failed to mlx5_modify_nic_vport_mac vport(%d) err=(%d)\n",
2124			       vport_num, err);
2125		return err;
2126	}
2127
2128	node_guid_gen_from_mac(&node_guid, mac);
2129	err = mlx5_modify_nic_vport_node_guid(esw->dev, vport_num, node_guid);
2130	if (err)
2131		mlx5_core_warn(esw->dev,
2132			       "Failed to set vport %d node guid, err = %d. RDMA_CM will not function properly for this VF.\n",
2133			       vport_num, err);
2134
2135	ether_addr_copy(evport->info.mac, mac);
2136	evport->info.node_guid = node_guid;
2137	if (evport->enabled && esw->mode == MLX5_ESWITCH_LEGACY)
2138		err = esw_acl_ingress_lgcy_setup(esw, evport);
2139
2140	return err;
2141}
2142
2143int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw,
2144			       u16 vport, const u8 *mac)
2145{
2146	struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport);
2147	int err = 0;
2148
2149	if (IS_ERR(evport))
2150		return PTR_ERR(evport);
2151
2152	mutex_lock(&esw->state_lock);
2153	err = mlx5_esw_set_vport_mac_locked(esw, evport, mac);
2154	mutex_unlock(&esw->state_lock);
2155	return err;
2156}
2157
2158static bool mlx5_esw_check_port_type(struct mlx5_eswitch *esw, u16 vport_num, xa_mark_t mark)
2159{
2160	return xa_get_mark(&esw->vports, vport_num, mark);
2161}
2162
2163bool mlx5_eswitch_is_vf_vport(struct mlx5_eswitch *esw, u16 vport_num)
2164{
2165	return mlx5_esw_check_port_type(esw, vport_num, MLX5_ESW_VPT_VF);
2166}
2167
2168bool mlx5_eswitch_is_pf_vf_vport(struct mlx5_eswitch *esw, u16 vport_num)
2169{
2170	return vport_num == MLX5_VPORT_PF ||
2171		mlx5_eswitch_is_vf_vport(esw, vport_num);
2172}
2173
2174bool mlx5_esw_is_sf_vport(struct mlx5_eswitch *esw, u16 vport_num)
2175{
2176	return mlx5_esw_check_port_type(esw, vport_num, MLX5_ESW_VPT_SF);
2177}
2178
2179int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw,
2180				 u16 vport, int link_state)
2181{
2182	struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport);
2183	int opmod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT;
2184	int other_vport = 1;
2185	int err = 0;
2186
2187	if (!mlx5_esw_allowed(esw))
2188		return -EPERM;
2189	if (IS_ERR(evport))
2190		return PTR_ERR(evport);
2191
2192	if (vport == MLX5_VPORT_UPLINK) {
2193		opmod = MLX5_VPORT_STATE_OP_MOD_UPLINK;
2194		other_vport = 0;
2195		vport = 0;
2196	}
2197	mutex_lock(&esw->state_lock);
2198	if (esw->mode != MLX5_ESWITCH_LEGACY) {
2199		err = -EOPNOTSUPP;
2200		goto unlock;
2201	}
2202
2203	err = mlx5_modify_vport_admin_state(esw->dev, opmod, vport, other_vport, link_state);
2204	if (err) {
2205		mlx5_core_warn(esw->dev, "Failed to set vport %d link state, opmod = %d, err = %d",
2206			       vport, opmod, err);
2207		goto unlock;
2208	}
2209
2210	evport->info.link_state = link_state;
2211
2212unlock:
2213	mutex_unlock(&esw->state_lock);
2214	return err;
2215}
2216
2217int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
2218				  u16 vport, struct ifla_vf_info *ivi)
2219{
2220	struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport);
2221	u32 max_rate, min_rate;
2222
2223	if (IS_ERR(evport))
2224		return PTR_ERR(evport);
2225
2226	memset(ivi, 0, sizeof(*ivi));
2227	ivi->vf = vport - 1;
2228
2229	mutex_lock(&esw->state_lock);
2230
2231	mlx5_query_nic_vport_mac_address(esw->dev, vport, true,
2232					 evport->info.mac);
2233	ether_addr_copy(ivi->mac, evport->info.mac);
2234	ivi->linkstate = evport->info.link_state;
2235	ivi->vlan = evport->info.vlan;
2236	ivi->qos = evport->info.qos;
2237	ivi->spoofchk = evport->info.spoofchk;
2238	ivi->trusted = evport->info.trusted;
2239
2240	if (mlx5_esw_qos_get_vport_rate(evport, &max_rate, &min_rate)) {
2241		ivi->max_tx_rate = max_rate;
2242		ivi->min_tx_rate = min_rate;
2243	}
2244	mutex_unlock(&esw->state_lock);
2245
2246	return 0;
2247}
2248
2249int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw,
2250				  u16 vport, u16 vlan, u8 qos, u8 set_flags)
2251{
2252	struct mlx5_vport *evport = mlx5_eswitch_get_vport(esw, vport);
2253	bool vst_mode_steering = esw_vst_mode_is_steering(esw);
2254	int err = 0;
2255
2256	if (IS_ERR(evport))
2257		return PTR_ERR(evport);
2258	if (vlan > 4095 || qos > 7)
2259		return -EINVAL;
2260
2261	if (esw->mode == MLX5_ESWITCH_OFFLOADS || !vst_mode_steering) {
2262		err = modify_esw_vport_cvlan(esw->dev, vport, vlan, qos, set_flags);
2263		if (err)
2264			return err;
2265	}
2266
2267	evport->info.vlan = vlan;
2268	evport->info.qos = qos;
2269	if (evport->enabled && esw->mode == MLX5_ESWITCH_LEGACY) {
2270		err = esw_acl_ingress_lgcy_setup(esw, evport);
2271		if (err)
2272			return err;
2273		err = esw_acl_egress_lgcy_setup(esw, evport);
2274	}
2275
2276	return err;
2277}
2278
2279int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
2280				 u16 vport_num,
2281				 struct ifla_vf_stats *vf_stats)
2282{
2283	struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
2284	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
2285	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {};
2286	struct mlx5_vport_drop_stats stats = {};
2287	int err = 0;
2288	u32 *out;
2289
2290	if (IS_ERR(vport))
2291		return PTR_ERR(vport);
2292
2293	out = kvzalloc(outlen, GFP_KERNEL);
2294	if (!out)
2295		return -ENOMEM;
2296
2297	MLX5_SET(query_vport_counter_in, in, opcode,
2298		 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
2299	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
2300	MLX5_SET(query_vport_counter_in, in, vport_number, vport->vport);
2301	MLX5_SET(query_vport_counter_in, in, other_vport, 1);
2302
2303	err = mlx5_cmd_exec_inout(esw->dev, query_vport_counter, in, out);
2304	if (err)
2305		goto free_out;
2306
2307	#define MLX5_GET_CTR(p, x) \
2308		MLX5_GET64(query_vport_counter_out, p, x)
2309
2310	memset(vf_stats, 0, sizeof(*vf_stats));
2311	vf_stats->rx_packets =
2312		MLX5_GET_CTR(out, received_eth_unicast.packets) +
2313		MLX5_GET_CTR(out, received_ib_unicast.packets) +
2314		MLX5_GET_CTR(out, received_eth_multicast.packets) +
2315		MLX5_GET_CTR(out, received_ib_multicast.packets) +
2316		MLX5_GET_CTR(out, received_eth_broadcast.packets);
2317
2318	vf_stats->rx_bytes =
2319		MLX5_GET_CTR(out, received_eth_unicast.octets) +
2320		MLX5_GET_CTR(out, received_ib_unicast.octets) +
2321		MLX5_GET_CTR(out, received_eth_multicast.octets) +
2322		MLX5_GET_CTR(out, received_ib_multicast.octets) +
2323		MLX5_GET_CTR(out, received_eth_broadcast.octets);
2324
2325	vf_stats->tx_packets =
2326		MLX5_GET_CTR(out, transmitted_eth_unicast.packets) +
2327		MLX5_GET_CTR(out, transmitted_ib_unicast.packets) +
2328		MLX5_GET_CTR(out, transmitted_eth_multicast.packets) +
2329		MLX5_GET_CTR(out, transmitted_ib_multicast.packets) +
2330		MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
2331
2332	vf_stats->tx_bytes =
2333		MLX5_GET_CTR(out, transmitted_eth_unicast.octets) +
2334		MLX5_GET_CTR(out, transmitted_ib_unicast.octets) +
2335		MLX5_GET_CTR(out, transmitted_eth_multicast.octets) +
2336		MLX5_GET_CTR(out, transmitted_ib_multicast.octets) +
2337		MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
2338
2339	vf_stats->multicast =
2340		MLX5_GET_CTR(out, received_eth_multicast.packets) +
2341		MLX5_GET_CTR(out, received_ib_multicast.packets);
2342
2343	vf_stats->broadcast =
2344		MLX5_GET_CTR(out, received_eth_broadcast.packets);
2345
2346	err = mlx5_esw_query_vport_drop_stats(esw->dev, vport, &stats);
2347	if (err)
2348		goto free_out;
2349	vf_stats->rx_dropped = stats.rx_dropped;
2350	vf_stats->tx_dropped = stats.tx_dropped;
2351
2352free_out:
2353	kvfree(out);
2354	return err;
2355}
2356
2357u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev)
2358{
2359	struct mlx5_eswitch *esw = dev->priv.eswitch;
2360
2361	return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_LEGACY;
2362}
2363EXPORT_SYMBOL_GPL(mlx5_eswitch_mode);
2364
2365enum devlink_eswitch_encap_mode
2366mlx5_eswitch_get_encap_mode(const struct mlx5_core_dev *dev)
2367{
2368	struct mlx5_eswitch *esw;
2369
2370	esw = dev->priv.eswitch;
2371	return (mlx5_eswitch_mode(dev) == MLX5_ESWITCH_OFFLOADS)  ? esw->offloads.encap :
2372		DEVLINK_ESWITCH_ENCAP_MODE_NONE;
2373}
2374EXPORT_SYMBOL(mlx5_eswitch_get_encap_mode);
2375
2376bool mlx5_esw_multipath_prereq(struct mlx5_core_dev *dev0,
2377			       struct mlx5_core_dev *dev1)
2378{
2379	return (dev0->priv.eswitch->mode == MLX5_ESWITCH_OFFLOADS &&
2380		dev1->priv.eswitch->mode == MLX5_ESWITCH_OFFLOADS);
2381}
2382
2383int mlx5_esw_event_notifier_register(struct mlx5_core_dev *dev,
2384				     struct notifier_block *nb)
2385{
2386	return blocking_notifier_chain_register(&dev->priv.esw_n_head, nb);
2387}
2388
2389void mlx5_esw_event_notifier_unregister(struct mlx5_core_dev *dev,
2390					struct notifier_block *nb)
2391{
2392	blocking_notifier_chain_unregister(&dev->priv.esw_n_head, nb);
2393}
2394
2395/**
2396 * mlx5_esw_hold() - Try to take a read lock on esw mode lock.
2397 * @mdev: mlx5 core device.
2398 *
2399 * Should be called by esw resources callers.
2400 *
2401 * Return: true on success or false.
2402 */
2403bool mlx5_esw_hold(struct mlx5_core_dev *mdev)
2404{
2405	struct mlx5_eswitch *esw = mdev->priv.eswitch;
2406
2407	/* e.g. VF doesn't have eswitch so nothing to do */
2408	if (!mlx5_esw_allowed(esw))
2409		return true;
2410
2411	if (down_read_trylock(&esw->mode_lock) != 0) {
2412		if (esw->eswitch_operation_in_progress) {
2413			up_read(&esw->mode_lock);
2414			return false;
2415		}
2416		return true;
2417	}
2418
2419	return false;
2420}
2421
2422/**
2423 * mlx5_esw_release() - Release a read lock on esw mode lock.
2424 * @mdev: mlx5 core device.
2425 */
2426void mlx5_esw_release(struct mlx5_core_dev *mdev)
2427{
2428	struct mlx5_eswitch *esw = mdev->priv.eswitch;
2429
2430	if (mlx5_esw_allowed(esw))
2431		up_read(&esw->mode_lock);
2432}
2433
2434/**
2435 * mlx5_esw_get() - Increase esw user count.
2436 * @mdev: mlx5 core device.
2437 */
2438void mlx5_esw_get(struct mlx5_core_dev *mdev)
2439{
2440	struct mlx5_eswitch *esw = mdev->priv.eswitch;
2441
2442	if (mlx5_esw_allowed(esw))
2443		atomic64_inc(&esw->user_count);
2444}
2445
2446/**
2447 * mlx5_esw_put() - Decrease esw user count.
2448 * @mdev: mlx5 core device.
2449 */
2450void mlx5_esw_put(struct mlx5_core_dev *mdev)
2451{
2452	struct mlx5_eswitch *esw = mdev->priv.eswitch;
2453
2454	if (mlx5_esw_allowed(esw))
2455		atomic64_dec_if_positive(&esw->user_count);
2456}
2457
2458/**
2459 * mlx5_esw_try_lock() - Take a write lock on esw mode lock.
2460 * @esw: eswitch device.
2461 *
2462 * Should be called by esw mode change routine.
2463 *
2464 * Return:
2465 * * 0       - esw mode if successfully locked and refcount is 0.
2466 * * -EBUSY  - refcount is not 0.
2467 * * -EINVAL - In the middle of switching mode or lock is already held.
2468 */
2469int mlx5_esw_try_lock(struct mlx5_eswitch *esw)
2470{
2471	if (down_write_trylock(&esw->mode_lock) == 0)
2472		return -EINVAL;
2473
2474	if (esw->eswitch_operation_in_progress ||
2475	    atomic64_read(&esw->user_count) > 0) {
2476		up_write(&esw->mode_lock);
2477		return -EBUSY;
2478	}
2479
2480	return esw->mode;
2481}
2482
2483int mlx5_esw_lock(struct mlx5_eswitch *esw)
2484{
2485	down_write(&esw->mode_lock);
2486
2487	if (esw->eswitch_operation_in_progress) {
2488		up_write(&esw->mode_lock);
2489		return -EBUSY;
2490	}
2491
2492	return 0;
2493}
2494
2495/**
2496 * mlx5_esw_unlock() - Release write lock on esw mode lock
2497 * @esw: eswitch device.
2498 */
2499void mlx5_esw_unlock(struct mlx5_eswitch *esw)
2500{
2501	up_write(&esw->mode_lock);
2502}
2503
2504/**
2505 * mlx5_eswitch_get_total_vports - Get total vports of the eswitch
2506 *
2507 * @dev: Pointer to core device
2508 *
2509 * mlx5_eswitch_get_total_vports returns total number of eswitch vports.
2510 */
2511u16 mlx5_eswitch_get_total_vports(const struct mlx5_core_dev *dev)
2512{
2513	struct mlx5_eswitch *esw;
2514
2515	esw = dev->priv.eswitch;
2516	return mlx5_esw_allowed(esw) ? esw->total_vports : 0;
2517}
2518EXPORT_SYMBOL_GPL(mlx5_eswitch_get_total_vports);
2519
2520/**
2521 * mlx5_eswitch_get_core_dev - Get the mdev device
2522 * @esw : eswitch device.
2523 *
2524 * Return the mellanox core device which manages the eswitch.
2525 */
2526struct mlx5_core_dev *mlx5_eswitch_get_core_dev(struct mlx5_eswitch *esw)
2527{
2528	return mlx5_esw_allowed(esw) ? esw->dev : NULL;
2529}
2530EXPORT_SYMBOL(mlx5_eswitch_get_core_dev);
2531
2532bool mlx5_eswitch_block_ipsec(struct mlx5_core_dev *dev)
2533{
2534	struct mlx5_eswitch *esw = dev->priv.eswitch;
2535
2536	if (!mlx5_esw_allowed(esw))
2537		return true;
2538
2539	mutex_lock(&esw->state_lock);
2540	if (esw->enabled_ipsec_vf_count) {
2541		mutex_unlock(&esw->state_lock);
2542		return false;
2543	}
2544
2545	dev->num_ipsec_offloads++;
2546	mutex_unlock(&esw->state_lock);
2547	return true;
2548}
2549
2550void mlx5_eswitch_unblock_ipsec(struct mlx5_core_dev *dev)
2551{
2552	struct mlx5_eswitch *esw = dev->priv.eswitch;
2553
2554	if (!mlx5_esw_allowed(esw))
2555		/* Failure means no eswitch => core dev is not a PF */
2556		return;
2557
2558	mutex_lock(&esw->state_lock);
2559	dev->num_ipsec_offloads--;
2560	mutex_unlock(&esw->state_lock);
2561}
2562
2563bool mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev)
2564{
2565	if (!dev->priv.eswitch)
2566		return true;
2567
2568	return !dev->priv.eswitch->esw_funcs.host_funcs_disabled;
2569}