en_rep.c

   1/*
   2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/dim.h>
  34#include <linux/debugfs.h>
  35#include <linux/mlx5/fs.h>
  36#include <net/netdev_lock.h>
  37#include <net/switchdev.h>
  38#include <net/pkt_cls.h>
  39#include <net/act_api.h>
  40#include <net/devlink.h>
  41#include <net/ipv6_stubs.h>
  42
  43#include "eswitch.h"
  44#include "en.h"
  45#include "en/dim.h"
  46#include "en_rep.h"
  47#include "en/params.h"
  48#include "en/txrx.h"
  49#include "en_tc.h"
  50#include "en/rep/tc.h"
  51#include "en/rep/neigh.h"
  52#include "en/rep/bridge.h"
  53#include "en/devlink.h"
  54#include "fs_core.h"
  55#include "lib/mlx5.h"
  56#include "lib/devcom.h"
  57#include "lib/vxlan.h"
  58#define CREATE_TRACE_POINTS
  59#include "diag/en_rep_tracepoint.h"
  60#include "diag/reporter_vnic.h"
  61#include "en_accel/ipsec.h"
  62#include "en/tc/int_port.h"
  63#include "en/ptp.h"
  64#include "en/fs_ethtool.h"
  65
  66#define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \
  67	max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
  68#define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1
  69#define MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE 0x8
  70
  71static const char mlx5e_rep_driver_name[] = "mlx5e_rep";
  72
  73static void mlx5e_rep_get_drvinfo(struct net_device *dev,
  74				  struct ethtool_drvinfo *drvinfo)
  75{
  76	struct mlx5e_priv *priv = netdev_priv(dev);
  77	struct mlx5_core_dev *mdev = priv->mdev;
  78	int count;
  79
  80	strscpy(drvinfo->driver, mlx5e_rep_driver_name,
  81		sizeof(drvinfo->driver));
  82	count = snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
  83			 "%d.%d.%04d (%.16s)", fw_rev_maj(mdev),
  84			 fw_rev_min(mdev), fw_rev_sub(mdev), mdev->board_id);
  85	if (count >= sizeof(drvinfo->fw_version))
  86		snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
  87			 "%d.%d.%04d", fw_rev_maj(mdev),
  88			 fw_rev_min(mdev), fw_rev_sub(mdev));
  89}
  90
  91static const struct counter_desc sw_rep_stats_desc[] = {
  92	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) },
  93	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) },
  94	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) },
  95	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) },
  96};
  97
  98static const struct counter_desc vport_rep_stats_desc[] = {
  99	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_packets) },
 100	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_bytes) },
 101	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_packets) },
 102	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_bytes) },
 103	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 104			     rx_vport_rdma_unicast_packets) },
 105	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, rx_vport_rdma_unicast_bytes) },
 106	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 107			     tx_vport_rdma_unicast_packets) },
 108	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, tx_vport_rdma_unicast_bytes) },
 109	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 110			     rx_vport_rdma_multicast_packets) },
 111	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 112			     rx_vport_rdma_multicast_bytes) },
 113	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 114			     tx_vport_rdma_multicast_packets) },
 115	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 116			     tx_vport_rdma_multicast_bytes) },
 117};
 118
 119static const struct counter_desc vport_rep_loopback_stats_desc[] = {
 120	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 121			     vport_loopback_packets) },
 122	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
 123			     vport_loopback_bytes) },
 124};
 125
 126#define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
 127#define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
 128#define NUM_VPORT_REP_LOOPBACK_COUNTERS(dev) \
 129	(MLX5_CAP_GEN(dev, vport_counter_local_loopback) ? \
 130	 ARRAY_SIZE(vport_rep_loopback_stats_desc) : 0)
 131
 132static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep)
 133{
 134	return NUM_VPORT_REP_SW_COUNTERS;
 135}
 136
 137static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep)
 138{
 139	int i;
 140
 141	for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 142		ethtool_puts(data, sw_rep_stats_desc[i].format);
 143}
 144
 145static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep)
 146{
 147	int i;
 148
 149	for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++)
 150		mlx5e_ethtool_put_stat(
 151			data, MLX5E_READ_CTR64_CPU(&priv->stats.sw,
 152						   sw_rep_stats_desc, i));
 153}
 154
 155static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep)
 156{
 157	struct mlx5e_sw_stats *s = &priv->stats.sw;
 158	struct rtnl_link_stats64 stats64 = {};
 159
 160	memset(s, 0, sizeof(*s));
 161	mlx5e_fold_sw_stats64(priv, &stats64);
 162
 163	s->rx_packets = stats64.rx_packets;
 164	s->rx_bytes   = stats64.rx_bytes;
 165	s->tx_packets = stats64.tx_packets;
 166	s->tx_bytes   = stats64.tx_bytes;
 167	s->tx_queue_dropped = stats64.tx_dropped;
 168}
 169
 170static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep)
 171{
 172	return NUM_VPORT_REP_HW_COUNTERS +
 173	       NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev);
 174}
 175
 176static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep)
 177{
 178	int i;
 179
 180	for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 181		ethtool_puts(data, vport_rep_stats_desc[i].format);
 182	for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
 183		ethtool_puts(data, vport_rep_loopback_stats_desc[i].format);
 184}
 185
 186static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep)
 187{
 188	int i;
 189
 190	for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 191		mlx5e_ethtool_put_stat(
 192			data, MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
 193						   vport_rep_stats_desc, i));
 194	for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
 195		mlx5e_ethtool_put_stat(
 196			data,
 197			MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
 198					     vport_rep_loopback_stats_desc, i));
 199}
 200
 201static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep)
 202{
 203	struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats;
 204	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 205	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 206	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 207	struct mlx5_eswitch_rep *rep = rpriv->rep;
 208	u32 *out;
 209	int err;
 210
 211	out = kvzalloc(outlen, GFP_KERNEL);
 212	if (!out)
 213		return;
 214
 215	err = mlx5_core_query_vport_counter(esw->dev, 1, rep->vport - 1, 0, out);
 216	if (err) {
 217		netdev_warn(priv->netdev, "vport %d error %d reading stats\n",
 218			    rep->vport, err);
 219		goto out;
 220	}
 221
 222	#define MLX5_GET_CTR(p, x) \
 223		MLX5_GET64(query_vport_counter_out, p, x)
 224	/* flip tx/rx as we are reporting the counters for the switch vport */
 225	rep_stats->vport_rx_packets =
 226		MLX5_GET_CTR(out, transmitted_ib_unicast.packets) +
 227		MLX5_GET_CTR(out, transmitted_eth_unicast.packets) +
 228		MLX5_GET_CTR(out, transmitted_ib_multicast.packets) +
 229		MLX5_GET_CTR(out, transmitted_eth_multicast.packets) +
 230		MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
 231
 232	rep_stats->vport_tx_packets =
 233		MLX5_GET_CTR(out, received_ib_unicast.packets) +
 234		MLX5_GET_CTR(out, received_eth_unicast.packets) +
 235		MLX5_GET_CTR(out, received_ib_multicast.packets) +
 236		MLX5_GET_CTR(out, received_eth_multicast.packets) +
 237		MLX5_GET_CTR(out, received_eth_broadcast.packets);
 238
 239	rep_stats->vport_rx_bytes =
 240		MLX5_GET_CTR(out, transmitted_ib_unicast.octets) +
 241		MLX5_GET_CTR(out, transmitted_eth_unicast.octets) +
 242		MLX5_GET_CTR(out, transmitted_ib_multicast.octets) +
 243		MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
 244
 245	rep_stats->vport_tx_bytes =
 246		MLX5_GET_CTR(out, received_ib_unicast.octets) +
 247		MLX5_GET_CTR(out, received_eth_unicast.octets) +
 248		MLX5_GET_CTR(out, received_ib_multicast.octets) +
 249		MLX5_GET_CTR(out, received_eth_multicast.octets) +
 250		MLX5_GET_CTR(out, received_eth_broadcast.octets);
 251
 252	rep_stats->rx_vport_rdma_unicast_packets =
 253		MLX5_GET_CTR(out, transmitted_ib_unicast.packets);
 254	rep_stats->tx_vport_rdma_unicast_packets =
 255		MLX5_GET_CTR(out, received_ib_unicast.packets);
 256	rep_stats->rx_vport_rdma_unicast_bytes =
 257		MLX5_GET_CTR(out, transmitted_ib_unicast.octets);
 258	rep_stats->tx_vport_rdma_unicast_bytes =
 259		MLX5_GET_CTR(out, received_ib_unicast.octets);
 260	rep_stats->rx_vport_rdma_multicast_packets =
 261		MLX5_GET_CTR(out, transmitted_ib_multicast.packets);
 262	rep_stats->tx_vport_rdma_multicast_packets =
 263		MLX5_GET_CTR(out, received_ib_multicast.packets);
 264	rep_stats->rx_vport_rdma_multicast_bytes =
 265		MLX5_GET_CTR(out, transmitted_ib_multicast.octets);
 266	rep_stats->tx_vport_rdma_multicast_bytes =
 267		MLX5_GET_CTR(out, received_ib_multicast.octets);
 268
 269	if (MLX5_CAP_GEN(priv->mdev, vport_counter_local_loopback)) {
 270		rep_stats->vport_loopback_packets =
 271			MLX5_GET_CTR(out, local_loopback.packets);
 272		rep_stats->vport_loopback_bytes =
 273			MLX5_GET_CTR(out, local_loopback.octets);
 274	}
 275
 276out:
 277	kvfree(out);
 278}
 279
 280static int mlx5e_rep_query_aggr_q_counter(struct mlx5_core_dev *dev, int vport, void *out)
 281{
 282	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
 283
 284	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
 285	MLX5_SET(query_q_counter_in, in, other_vport, 1);
 286	MLX5_SET(query_q_counter_in, in, vport_number, vport);
 287	MLX5_SET(query_q_counter_in, in, aggregate, 1);
 288
 289	return mlx5_cmd_exec_inout(dev, query_q_counter, in, out);
 290}
 291
 292static void mlx5e_rep_update_vport_q_counter(struct mlx5e_priv *priv)
 293{
 294	struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats;
 295	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
 296	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 297	struct mlx5_eswitch_rep *rep = rpriv->rep;
 298	int err;
 299
 300	if (!MLX5_CAP_GEN(priv->mdev, q_counter_other_vport) ||
 301	    !MLX5_CAP_GEN(priv->mdev, q_counter_aggregation))
 302		return;
 303
 304	err = mlx5e_rep_query_aggr_q_counter(priv->mdev, rep->vport, out);
 305	if (err) {
 306		netdev_warn(priv->netdev, "failed reading stats on vport %d, error %d\n",
 307			    rep->vport, err);
 308		return;
 309	}
 310
 311	rep_stats->rx_vport_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
 312}
 313
 314static void mlx5e_rep_get_strings(struct net_device *dev,
 315				  u32 stringset, u8 *data)
 316{
 317	struct mlx5e_priv *priv = netdev_priv(dev);
 318
 319	switch (stringset) {
 320	case ETH_SS_STATS:
 321		mlx5e_stats_fill_strings(priv, data);
 322		break;
 323	}
 324}
 325
 326static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
 327					struct ethtool_stats *stats, u64 *data)
 328{
 329	struct mlx5e_priv *priv = netdev_priv(dev);
 330
 331	mlx5e_ethtool_get_ethtool_stats(priv, stats, data);
 332}
 333
 334static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset)
 335{
 336	struct mlx5e_priv *priv = netdev_priv(dev);
 337
 338	switch (sset) {
 339	case ETH_SS_STATS:
 340		return mlx5e_stats_total_num(priv);
 341	default:
 342		return -EOPNOTSUPP;
 343	}
 344}
 345
 346static void
 347mlx5e_rep_get_ringparam(struct net_device *dev,
 348			struct ethtool_ringparam *param,
 349			struct kernel_ethtool_ringparam *kernel_param,
 350			struct netlink_ext_ack *extack)
 351{
 352	struct mlx5e_priv *priv = netdev_priv(dev);
 353
 354	mlx5e_ethtool_get_ringparam(priv, param, kernel_param);
 355}
 356
 357static int
 358mlx5e_rep_set_ringparam(struct net_device *dev,
 359			struct ethtool_ringparam *param,
 360			struct kernel_ethtool_ringparam *kernel_param,
 361			struct netlink_ext_ack *extack)
 362{
 363	struct mlx5e_priv *priv = netdev_priv(dev);
 364
 365	return mlx5e_ethtool_set_ringparam(priv, param, extack);
 366}
 367
 368static void mlx5e_rep_get_channels(struct net_device *dev,
 369				   struct ethtool_channels *ch)
 370{
 371	struct mlx5e_priv *priv = netdev_priv(dev);
 372
 373	mlx5e_ethtool_get_channels(priv, ch);
 374}
 375
 376static int mlx5e_rep_set_channels(struct net_device *dev,
 377				  struct ethtool_channels *ch)
 378{
 379	struct mlx5e_priv *priv = netdev_priv(dev);
 380
 381	return mlx5e_ethtool_set_channels(priv, ch);
 382}
 383
 384static int mlx5e_rep_get_coalesce(struct net_device *netdev,
 385				  struct ethtool_coalesce *coal,
 386				  struct kernel_ethtool_coalesce *kernel_coal,
 387				  struct netlink_ext_ack *extack)
 388{
 389	struct mlx5e_priv *priv = netdev_priv(netdev);
 390
 391	return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal, extack);
 392}
 393
 394static int mlx5e_rep_set_coalesce(struct net_device *netdev,
 395				  struct ethtool_coalesce *coal,
 396				  struct kernel_ethtool_coalesce *kernel_coal,
 397				  struct netlink_ext_ack *extack)
 398{
 399	struct mlx5e_priv *priv = netdev_priv(netdev);
 400
 401	return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack);
 402}
 403
 404static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev)
 405{
 406	struct mlx5e_priv *priv = netdev_priv(netdev);
 407
 408	return mlx5e_ethtool_get_rxfh_key_size(priv);
 409}
 410
 411static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev)
 412{
 413	struct mlx5e_priv *priv = netdev_priv(netdev);
 414
 415	return mlx5e_ethtool_get_rxfh_indir_size(priv);
 416}
 417
 418static const struct ethtool_ops mlx5e_rep_ethtool_ops = {
 419	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 420				     ETHTOOL_COALESCE_MAX_FRAMES |
 421				     ETHTOOL_COALESCE_USE_ADAPTIVE,
 422	.get_drvinfo	   = mlx5e_rep_get_drvinfo,
 423	.get_link	   = ethtool_op_get_link,
 424	.get_strings       = mlx5e_rep_get_strings,
 425	.get_sset_count    = mlx5e_rep_get_sset_count,
 426	.get_ethtool_stats = mlx5e_rep_get_ethtool_stats,
 427	.get_ringparam     = mlx5e_rep_get_ringparam,
 428	.set_ringparam     = mlx5e_rep_set_ringparam,
 429	.get_channels      = mlx5e_rep_get_channels,
 430	.set_channels      = mlx5e_rep_set_channels,
 431	.get_coalesce      = mlx5e_rep_get_coalesce,
 432	.set_coalesce      = mlx5e_rep_set_coalesce,
 433	.get_per_queue_coalesce = mlx5e_get_per_queue_coalesce,
 434	.set_per_queue_coalesce = mlx5e_set_per_queue_coalesce,
 435	.get_rxfh_key_size   = mlx5e_rep_get_rxfh_key_size,
 436	.get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size,
 437};
 438
 439static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw,
 440				 struct mlx5_eswitch_rep *rep)
 441{
 442	struct mlx5e_rep_sq *rep_sq, *tmp;
 443	struct mlx5e_rep_sq_peer *sq_peer;
 444	struct mlx5e_rep_priv *rpriv;
 445	unsigned long i;
 446
 447	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
 448		return;
 449
 450	rpriv = mlx5e_rep_to_rep_priv(rep);
 451	list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) {
 452		mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule);
 453		xa_for_each(&rep_sq->sq_peer, i, sq_peer) {
 454			if (sq_peer->rule)
 455				mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule);
 456
 457			xa_erase(&rep_sq->sq_peer, i);
 458			kfree(sq_peer);
 459		}
 460
 461		xa_destroy(&rep_sq->sq_peer);
 462		list_del(&rep_sq->list);
 463		kfree(rep_sq);
 464	}
 465}
 466
 467static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep,
 468					   struct mlx5e_rep_sq *rep_sq, int i)
 469{
 470	struct mlx5_flow_handle *flow_rule;
 471	struct mlx5_devcom_comp_dev *tmp;
 472	struct mlx5_eswitch *peer_esw;
 473
 474	mlx5_devcom_for_each_peer_entry(esw->devcom, peer_esw, tmp) {
 475		u16 peer_rule_idx = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
 476		struct mlx5e_rep_sq_peer *sq_peer;
 477		int err;
 478
 479		sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL);
 480		if (!sq_peer)
 481			return -ENOMEM;
 482
 483		flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw,
 484								rep, rep_sq->sqn);
 485		if (IS_ERR(flow_rule)) {
 486			kfree(sq_peer);
 487			return PTR_ERR(flow_rule);
 488		}
 489
 490		sq_peer->rule = flow_rule;
 491		sq_peer->peer = peer_esw;
 492		err = xa_insert(&rep_sq->sq_peer, peer_rule_idx, sq_peer, GFP_KERNEL);
 493		if (err) {
 494			kfree(sq_peer);
 495			mlx5_eswitch_del_send_to_vport_rule(flow_rule);
 496			return err;
 497		}
 498	}
 499
 500	return 0;
 501}
 502
 503static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw,
 504				 struct mlx5_eswitch_rep *rep,
 505				 u32 *sqns_array, int sqns_num)
 506{
 507	struct mlx5_flow_handle *flow_rule;
 508	struct mlx5e_rep_priv *rpriv;
 509	struct mlx5e_rep_sq *rep_sq;
 510	bool devcom_locked = false;
 511	int err;
 512	int i;
 513
 514	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
 515		return 0;
 516
 517	rpriv = mlx5e_rep_to_rep_priv(rep);
 518
 519	if (mlx5_devcom_comp_is_ready(esw->devcom) &&
 520	    mlx5_devcom_for_each_peer_begin(esw->devcom))
 521		devcom_locked = true;
 522
 523	for (i = 0; i < sqns_num; i++) {
 524		rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL);
 525		if (!rep_sq) {
 526			err = -ENOMEM;
 527			goto out_err;
 528		}
 529
 530		/* Add re-inject rule to the PF/representor sqs */
 531		flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw, esw, rep,
 532								sqns_array[i]);
 533		if (IS_ERR(flow_rule)) {
 534			err = PTR_ERR(flow_rule);
 535			kfree(rep_sq);
 536			goto out_err;
 537		}
 538		rep_sq->send_to_vport_rule = flow_rule;
 539		rep_sq->sqn = sqns_array[i];
 540
 541		xa_init(&rep_sq->sq_peer);
 542		if (devcom_locked) {
 543			err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i);
 544			if (err) {
 545				mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule);
 546				xa_destroy(&rep_sq->sq_peer);
 547				kfree(rep_sq);
 548				goto out_err;
 549			}
 550		}
 551
 552		list_add(&rep_sq->list, &rpriv->vport_sqs_list);
 553	}
 554
 555	if (devcom_locked)
 556		mlx5_devcom_for_each_peer_end(esw->devcom);
 557
 558	return 0;
 559
 560out_err:
 561	mlx5e_sqs2vport_stop(esw, rep);
 562
 563	if (devcom_locked)
 564		mlx5_devcom_for_each_peer_end(esw->devcom);
 565
 566	return err;
 567}
 568
 569static int
 570mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 571{
 572	int sqs_per_channel = mlx5e_get_dcb_num_tc(&priv->channels.params);
 573	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 574	bool is_uplink_rep = mlx5e_is_uplink_rep(priv);
 575	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 576	struct mlx5_eswitch_rep *rep = rpriv->rep;
 577	int n, tc, nch, num_sqs = 0;
 578	struct mlx5e_channel *c;
 579	int err = -ENOMEM;
 580	bool ptp_sq;
 581	u32 *sqs;
 582
 583	ptp_sq = !!(priv->channels.ptp &&
 584		    MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS));
 585	nch = priv->channels.num + ptp_sq;
 586	/* +2 for xdpsqs, they don't exist on the ptp channel but will not be
 587	 * counted for by num_sqs.
 588	 */
 589	if (is_uplink_rep)
 590		sqs_per_channel += 2;
 591
 592	sqs = kvcalloc(nch * sqs_per_channel, sizeof(*sqs), GFP_KERNEL);
 593	if (!sqs)
 594		goto out;
 595
 596	for (n = 0; n < priv->channels.num; n++) {
 597		c = priv->channels.c[n];
 598		for (tc = 0; tc < c->num_tc; tc++)
 599			sqs[num_sqs++] = c->sq[tc].sqn;
 600
 601		if (is_uplink_rep) {
 602			if (c->xdp)
 603				sqs[num_sqs++] = c->rq_xdpsq.sqn;
 604
 605			if (c->xdpsq)
 606				sqs[num_sqs++] = c->xdpsq->sqn;
 607		}
 608	}
 609	if (ptp_sq) {
 610		struct mlx5e_ptp *ptp_ch = priv->channels.ptp;
 611
 612		for (tc = 0; tc < ptp_ch->num_tc; tc++)
 613			sqs[num_sqs++] = ptp_ch->ptpsq[tc].txqsq.sqn;
 614	}
 615
 616	err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs);
 617	kvfree(sqs);
 618
 619out:
 620	if (err)
 621		netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err);
 622	return err;
 623}
 624
 625static void
 626mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
 627{
 628	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 629	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 630	struct mlx5_eswitch_rep *rep = rpriv->rep;
 631
 632	mlx5e_sqs2vport_stop(esw, rep);
 633}
 634
 635static int
 636mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv)
 637{
 638	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 639	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 640	struct mlx5_eswitch_rep *rep = rpriv->rep;
 641	struct mlx5_flow_handle *flow_rule;
 642	struct mlx5_flow_group *g;
 643
 644	g = esw->fdb_table.offloads.send_to_vport_meta_grp;
 645	if (!g)
 646		return 0;
 647
 648	flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, rep->vport);
 649	if (IS_ERR(flow_rule))
 650		return PTR_ERR(flow_rule);
 651
 652	rpriv->send_to_vport_meta_rule = flow_rule;
 653
 654	return 0;
 655}
 656
 657static void
 658mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv)
 659{
 660	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 661
 662	if (rpriv->send_to_vport_meta_rule)
 663		mlx5_eswitch_del_send_to_vport_meta_rule(rpriv->send_to_vport_meta_rule);
 664}
 665
 666void mlx5e_rep_activate_channels(struct mlx5e_priv *priv)
 667{
 668	mlx5e_add_sqs_fwd_rules(priv);
 669	mlx5e_rep_add_meta_tunnel_rule(priv);
 670}
 671
 672void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv)
 673{
 674	mlx5e_rep_del_meta_tunnel_rule(priv);
 675	mlx5e_remove_sqs_fwd_rules(priv);
 676}
 677
 678static int mlx5e_rep_open(struct net_device *dev)
 679{
 680	struct mlx5e_priv *priv = netdev_priv(dev);
 681	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 682	struct mlx5_eswitch_rep *rep = rpriv->rep;
 683	int err;
 684
 685	mutex_lock(&priv->state_lock);
 686	err = mlx5e_open_locked(dev);
 687	if (err)
 688		goto unlock;
 689
 690	if (!mlx5_modify_vport_admin_state(priv->mdev,
 691					   MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 692					   rep->vport, 1,
 693					   MLX5_VPORT_ADMIN_STATE_UP))
 694		netif_carrier_on(dev);
 695
 696unlock:
 697	mutex_unlock(&priv->state_lock);
 698	return err;
 699}
 700
 701static int mlx5e_rep_close(struct net_device *dev)
 702{
 703	struct mlx5e_priv *priv = netdev_priv(dev);
 704	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 705	struct mlx5_eswitch_rep *rep = rpriv->rep;
 706	int ret;
 707
 708	mutex_lock(&priv->state_lock);
 709	mlx5_modify_vport_admin_state(priv->mdev,
 710				      MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 711				      rep->vport, 1,
 712				      MLX5_VPORT_ADMIN_STATE_DOWN);
 713	ret = mlx5e_close_locked(dev);
 714	mutex_unlock(&priv->state_lock);
 715	return ret;
 716}
 717
 718bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv)
 719{
 720	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 721	struct mlx5_eswitch_rep *rep;
 722
 723	if (!MLX5_ESWITCH_MANAGER(priv->mdev))
 724		return false;
 725
 726	if (!rpriv) /* non vport rep mlx5e instances don't use this field */
 727		return false;
 728
 729	rep = rpriv->rep;
 730	return (rep->vport == MLX5_VPORT_UPLINK);
 731}
 732
 733bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id)
 734{
 735	switch (attr_id) {
 736	case IFLA_OFFLOAD_XSTATS_CPU_HIT:
 737			return true;
 738	}
 739
 740	return false;
 741}
 742
 743static int
 744mlx5e_get_sw_stats64(const struct net_device *dev,
 745		     struct rtnl_link_stats64 *stats)
 746{
 747	struct mlx5e_priv *priv = netdev_priv(dev);
 748
 749	mlx5e_fold_sw_stats64(priv, stats);
 750	return 0;
 751}
 752
 753int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev,
 754				void *sp)
 755{
 756	switch (attr_id) {
 757	case IFLA_OFFLOAD_XSTATS_CPU_HIT:
 758		return mlx5e_get_sw_stats64(dev, sp);
 759	}
 760
 761	return -EINVAL;
 762}
 763
 764static void
 765mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
 766{
 767	struct mlx5e_priv *priv = netdev_priv(dev);
 768
 769	/* update HW stats in background for next time */
 770	mlx5e_queue_update_stats(priv);
 771	mlx5e_stats_copy_rep_stats(stats, &priv->stats.rep_stats);
 772}
 773
 774static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu)
 775{
 776	return mlx5e_change_mtu(netdev, new_mtu, NULL);
 777}
 778
 779static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier)
 780{
 781	struct mlx5e_priv *priv = netdev_priv(dev);
 782	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 783	struct mlx5_eswitch_rep *rep = rpriv->rep;
 784	int err;
 785
 786	if (new_carrier) {
 787		err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 788						    rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP);
 789		if (err)
 790			return err;
 791		netif_carrier_on(dev);
 792	} else {
 793		err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
 794						    rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN);
 795		if (err)
 796			return err;
 797		netif_carrier_off(dev);
 798	}
 799	return 0;
 800}
 801
 802static const struct net_device_ops mlx5e_netdev_ops_rep = {
 803	.ndo_open                = mlx5e_rep_open,
 804	.ndo_stop                = mlx5e_rep_close,
 805	.ndo_start_xmit          = mlx5e_xmit,
 806	.ndo_setup_tc            = mlx5e_rep_setup_tc,
 807	.ndo_set_mac_address	 = eth_mac_addr,
 808	.ndo_get_stats64         = mlx5e_rep_get_stats,
 809	.ndo_has_offload_stats	 = mlx5e_rep_has_offload_stats,
 810	.ndo_get_offload_stats	 = mlx5e_rep_get_offload_stats,
 811	.ndo_change_mtu          = mlx5e_rep_change_mtu,
 812	.ndo_change_carrier      = mlx5e_rep_change_carrier,
 813};
 814
 815bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev)
 816{
 817	return netdev->netdev_ops == &mlx5e_netdev_ops &&
 818	       mlx5e_is_uplink_rep(netdev_priv(netdev));
 819}
 820
 821bool mlx5e_eswitch_vf_rep(const struct net_device *netdev)
 822{
 823	return netdev->netdev_ops == &mlx5e_netdev_ops_rep;
 824}
 825
 826/* One indirect TIR set for outer. Inner not supported in reps. */
 827#define REP_NUM_INDIR_TIRS MLX5E_NUM_INDIR_TIRS
 828
 829static int mlx5e_rep_max_nch_limit(struct mlx5_core_dev *mdev)
 830{
 831	int max_tir_num = 1 << MLX5_CAP_GEN(mdev, log_max_tir);
 832	int num_vports = mlx5_eswitch_get_total_vports(mdev);
 833
 834	return (max_tir_num - mlx5e_get_pf_num_tirs(mdev)
 835		- (num_vports * REP_NUM_INDIR_TIRS)) / num_vports;
 836}
 837
 838static void mlx5e_build_rep_params(struct net_device *netdev)
 839{
 840	const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED;
 841	struct mlx5e_priv *priv = netdev_priv(netdev);
 842	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 843	struct mlx5_eswitch_rep *rep = rpriv->rep;
 844	struct mlx5_core_dev *mdev = priv->mdev;
 845	struct mlx5e_params *params;
 846
 847	params = &priv->channels.params;
 848
 849	params->num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS;
 850	params->hard_mtu    = MLX5E_ETH_HARD_MTU;
 851	params->sw_mtu      = netdev->mtu;
 852
 853	/* SQ */
 854	if (rep->vport == MLX5_VPORT_UPLINK)
 855		params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
 856	else
 857		params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE;
 858
 859	/* RQ */
 860	mlx5e_build_rq_params(mdev, params);
 861	if (!mlx5e_is_uplink_rep(priv) && mlx5_core_is_ecpf(mdev))
 862		params->log_rq_mtu_frames = MLX5E_REP_PARAMS_DEF_LOG_RQ_SIZE;
 863
 864	/* If netdev is already registered (e.g. move from nic profile to uplink,
 865	 * RTNL lock must be held before triggering netdev notifiers.
 866	 */
 867	if (take_rtnl)
 868		rtnl_lock();
 869	/* update XDP supported features */
 870	mlx5e_set_xdp_feature(priv);
 871	if (take_rtnl)
 872		rtnl_unlock();
 873
 874	/* CQ moderation params */
 875	params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
 876	params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe);
 877
 878	params->mqprio.num_tc       = 1;
 879	if (rep->vport != MLX5_VPORT_UPLINK)
 880		params->vlan_strip_disable = true;
 881
 882	mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
 883}
 884
 885static void mlx5e_build_rep_netdev(struct net_device *netdev,
 886				   struct mlx5_core_dev *mdev)
 887{
 888	SET_NETDEV_DEV(netdev, mdev->device);
 889	netdev->netdev_ops = &mlx5e_netdev_ops_rep;
 890	netdev->request_ops_lock = true;
 891	netdev_lockdep_set_classes(netdev);
 892	eth_hw_addr_random(netdev);
 893	netdev->ethtool_ops = &mlx5e_rep_ethtool_ops;
 894
 895	netdev->watchdog_timeo    = 15 * HZ;
 896	if (mlx5_core_is_ecpf(mdev))
 897		netdev->tx_queue_len = 1 << MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE;
 898
 899#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
 900	netdev->hw_features    |= NETIF_F_HW_TC;
 901#endif
 902	netdev->hw_features    |= NETIF_F_SG;
 903	netdev->hw_features    |= NETIF_F_IP_CSUM;
 904	netdev->hw_features    |= NETIF_F_IPV6_CSUM;
 905	netdev->hw_features    |= NETIF_F_GRO;
 906	netdev->hw_features    |= NETIF_F_TSO;
 907	netdev->hw_features    |= NETIF_F_TSO6;
 908	netdev->hw_features    |= NETIF_F_RXCSUM;
 909
 910	netdev->features |= netdev->hw_features;
 911
 912	netdev->netns_immutable = true;
 913}
 914
 915static int mlx5e_init_rep(struct mlx5_core_dev *mdev,
 916			  struct net_device *netdev)
 917{
 918	struct mlx5e_priv *priv = netdev_priv(netdev);
 919
 920	priv->fs =
 921		mlx5e_fs_init(priv->profile, mdev,
 922			      !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
 923			      priv->dfs_root);
 924	if (!priv->fs) {
 925		netdev_err(priv->netdev, "FS allocation failed\n");
 926		return -ENOMEM;
 927	}
 928
 929	mlx5e_build_rep_params(netdev);
 930	mlx5e_timestamp_init(priv);
 931
 932	return 0;
 933}
 934
 935static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev,
 936			     struct net_device *netdev)
 937{
 938	struct mlx5e_priv *priv = netdev_priv(netdev);
 939
 940	priv->dfs_root = debugfs_create_dir("nic",
 941					    mlx5_debugfs_get_dev_root(mdev));
 942
 943	priv->fs = mlx5e_fs_init(priv->profile, mdev,
 944				 !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
 945				 priv->dfs_root);
 946	if (!priv->fs) {
 947		netdev_err(priv->netdev, "FS allocation failed\n");
 948		debugfs_remove_recursive(priv->dfs_root);
 949		return -ENOMEM;
 950	}
 951
 952	mlx5e_vxlan_set_netdev_info(priv);
 953	mlx5e_build_rep_params(netdev);
 954	mlx5e_timestamp_init(priv);
 955	return 0;
 956}
 957
 958static void mlx5e_cleanup_rep(struct mlx5e_priv *priv)
 959{
 960	mlx5e_fs_cleanup(priv->fs);
 961	debugfs_remove_recursive(priv->dfs_root);
 962	priv->fs = NULL;
 963}
 964
 965static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv)
 966{
 967	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 968	struct mlx5_eswitch_rep *rep = rpriv->rep;
 969	struct ttc_params ttc_params = {};
 970	int err;
 971
 972	mlx5e_fs_set_ns(priv->fs,
 973			mlx5_get_flow_namespace(priv->mdev,
 974						MLX5_FLOW_NAMESPACE_KERNEL), false);
 975
 976	/* The inner_ttc in the ttc params is intentionally not set */
 977	mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false, false);
 978
 979	if (rep->vport != MLX5_VPORT_UPLINK)
 980		/* To give uplik rep TTC a lower level for chaining from root ft */
 981		ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1;
 982
 983	mlx5e_fs_set_ttc(priv->fs, mlx5_create_ttc_table(priv->mdev, &ttc_params), false);
 984	if (IS_ERR(mlx5e_fs_get_ttc(priv->fs, false))) {
 985		err = PTR_ERR(mlx5e_fs_get_ttc(priv->fs, false));
 986		netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n",
 987			   err);
 988		return err;
 989	}
 990	return 0;
 991}
 992
 993static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv)
 994{
 995	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 996	struct mlx5_eswitch_rep *rep = rpriv->rep;
 997	struct mlx5_flow_table_attr ft_attr = {};
 998	struct mlx5_flow_namespace *ns;
 999	int err = 0;
1000
1001	if (rep->vport != MLX5_VPORT_UPLINK) {
1002		/* non uplik reps will skip any bypass tables and go directly to
1003		 * their own ttc
1004		 */
1005		rpriv->root_ft = mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv->fs, false));
1006		return 0;
1007	}
1008
1009	/* uplink root ft will be used to auto chain, to ethtool or ttc tables */
1010	ns = mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1011	if (!ns) {
1012		netdev_err(priv->netdev, "Failed to get reps offloads namespace\n");
1013		return -EOPNOTSUPP;
1014	}
1015
1016	ft_attr.max_fte = 0; /* Empty table, miss rule will always point to next table */
1017	ft_attr.prio = 1;
1018	ft_attr.level = 1;
1019
1020	rpriv->root_ft = mlx5_create_flow_table(ns, &ft_attr);
1021	if (IS_ERR(rpriv->root_ft)) {
1022		err = PTR_ERR(rpriv->root_ft);
1023		rpriv->root_ft = NULL;
1024	}
1025
1026	return err;
1027}
1028
1029static void mlx5e_destroy_rep_root_ft(struct mlx5e_priv *priv)
1030{
1031	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1032	struct mlx5_eswitch_rep *rep = rpriv->rep;
1033
1034	if (rep->vport != MLX5_VPORT_UPLINK)
1035		return;
1036	mlx5_destroy_flow_table(rpriv->root_ft);
1037}
1038
1039static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv)
1040{
1041	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1042	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1043	struct mlx5_eswitch_rep *rep = rpriv->rep;
1044	struct mlx5_flow_handle *flow_rule;
1045	struct mlx5_flow_destination dest;
1046
1047	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1048	dest.ft = rpriv->root_ft;
1049
1050	flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, rep->vport, &dest);
1051	if (IS_ERR(flow_rule))
1052		return PTR_ERR(flow_rule);
1053	rpriv->vport_rx_rule = flow_rule;
1054	return 0;
1055}
1056
1057static void rep_vport_rx_rule_destroy(struct mlx5e_priv *priv)
1058{
1059	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1060
1061	if (!rpriv->vport_rx_rule)
1062		return;
1063
1064	mlx5_del_flow_rules(rpriv->vport_rx_rule);
1065	rpriv->vport_rx_rule = NULL;
1066}
1067
1068int mlx5e_rep_bond_update(struct mlx5e_priv *priv, bool cleanup)
1069{
1070	rep_vport_rx_rule_destroy(priv);
1071
1072	return cleanup ? 0 : mlx5e_create_rep_vport_rx_rule(priv);
1073}
1074
1075static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
1076{
1077	struct mlx5_core_dev *mdev = priv->mdev;
1078	int err;
1079
1080	mlx5e_fs_init_l2_addr(priv->fs, priv->netdev);
1081
1082	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
1083	if (err) {
1084		mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
1085		goto err_free_fs;
1086	}
1087
1088	priv->rx_res = mlx5e_rx_res_create(priv->mdev, 0, priv->max_nch, priv->drop_rq.rqn,
1089					   &priv->channels.params.packet_merge,
1090					   priv->channels.params.num_channels);
1091	if (IS_ERR(priv->rx_res)) {
1092		err = PTR_ERR(priv->rx_res);
1093		mlx5_core_err(mdev, "Create rx resources failed, err=%d\n", err);
1094		goto err_close_drop_rq;
1095	}
1096
1097	err = mlx5e_create_rep_ttc_table(priv);
1098	if (err)
1099		goto err_destroy_rx_res;
1100
1101	err = mlx5e_create_rep_root_ft(priv);
1102	if (err)
1103		goto err_destroy_ttc_table;
1104
1105	err = mlx5e_create_rep_vport_rx_rule(priv);
1106	if (err)
1107		goto err_destroy_root_ft;
1108
1109	mlx5e_ethtool_init_steering(priv->fs);
1110
1111	return 0;
1112
1113err_destroy_root_ft:
1114	mlx5e_destroy_rep_root_ft(priv);
1115err_destroy_ttc_table:
1116	mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false));
1117err_destroy_rx_res:
1118	mlx5e_rx_res_destroy(priv->rx_res);
1119	priv->rx_res = ERR_PTR(-EINVAL);
1120err_close_drop_rq:
1121	mlx5e_close_drop_rq(&priv->drop_rq);
1122err_free_fs:
1123	mlx5e_fs_cleanup(priv->fs);
1124	priv->fs = NULL;
1125	return err;
1126}
1127
1128static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
1129{
1130	mlx5e_ethtool_cleanup_steering(priv->fs);
1131	rep_vport_rx_rule_destroy(priv);
1132	mlx5e_destroy_rep_root_ft(priv);
1133	mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false));
1134	mlx5e_rx_res_destroy(priv->rx_res);
1135	priv->rx_res = ERR_PTR(-EINVAL);
1136	mlx5e_close_drop_rq(&priv->drop_rq);
1137}
1138
1139static void mlx5e_rep_mpesw_work(struct work_struct *work)
1140{
1141	struct mlx5_rep_uplink_priv *uplink_priv =
1142		container_of(work, struct mlx5_rep_uplink_priv,
1143			     mpesw_work);
1144	struct mlx5e_rep_priv *rpriv =
1145		container_of(uplink_priv, struct mlx5e_rep_priv,
1146			     uplink_priv);
1147	struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
1148
1149	rep_vport_rx_rule_destroy(priv);
1150	mlx5e_create_rep_vport_rx_rule(priv);
1151}
1152
1153static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv)
1154{
1155	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1156	int err;
1157
1158	mlx5e_create_q_counters(priv);
1159	err = mlx5e_init_rep_rx(priv);
1160	if (err)
1161		goto out;
1162
1163	mlx5e_tc_int_port_init_rep_rx(priv);
1164
1165	INIT_WORK(&rpriv->uplink_priv.mpesw_work, mlx5e_rep_mpesw_work);
1166
1167out:
1168	return err;
1169}
1170
1171static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv)
1172{
1173	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1174
1175	cancel_work_sync(&rpriv->uplink_priv.mpesw_work);
1176	mlx5e_tc_int_port_cleanup_rep_rx(priv);
1177	mlx5e_cleanup_rep_rx(priv);
1178	mlx5e_destroy_q_counters(priv);
1179}
1180
1181static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1182{
1183	struct mlx5_rep_uplink_priv *uplink_priv;
1184	struct net_device *netdev;
1185	struct mlx5e_priv *priv;
1186	int err;
1187
1188	netdev = rpriv->netdev;
1189	priv = netdev_priv(netdev);
1190	uplink_priv = &rpriv->uplink_priv;
1191
1192	err = mlx5e_rep_tc_init(rpriv);
1193	if (err)
1194		return err;
1195
1196	mlx5_init_port_tun_entropy(&uplink_priv->tun_entropy, priv->mdev);
1197
1198	mlx5e_rep_bond_init(rpriv);
1199	err = mlx5e_rep_tc_netdevice_event_register(rpriv);
1200	if (err) {
1201		mlx5_core_err(priv->mdev, "Failed to register netdev notifier, err: %d\n",
1202			      err);
1203		goto err_event_reg;
1204	}
1205
1206	return 0;
1207
1208err_event_reg:
1209	mlx5e_rep_bond_cleanup(rpriv);
1210	mlx5e_rep_tc_cleanup(rpriv);
1211	return err;
1212}
1213
1214static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv)
1215{
1216	mlx5e_rep_tc_netdevice_event_unregister(rpriv);
1217	mlx5e_rep_bond_cleanup(rpriv);
1218	mlx5e_rep_tc_cleanup(rpriv);
1219}
1220
1221static int mlx5e_init_rep_tx(struct mlx5e_priv *priv)
1222{
1223	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1224	int err;
1225
1226	err = mlx5e_rep_neigh_init(rpriv);
1227	if (err)
1228		goto err_neigh_init;
1229
1230	if (rpriv->rep->vport == MLX5_VPORT_UPLINK) {
1231		err = mlx5e_init_uplink_rep_tx(rpriv);
1232		if (err)
1233			goto err_init_tx;
1234	}
1235
1236	err = mlx5e_tc_ht_init(&rpriv->tc_ht);
1237	if (err)
1238		goto err_ht_init;
1239
1240	return 0;
1241
1242err_ht_init:
1243	if (rpriv->rep->vport == MLX5_VPORT_UPLINK)
1244		mlx5e_cleanup_uplink_rep_tx(rpriv);
1245err_init_tx:
1246	mlx5e_rep_neigh_cleanup(rpriv);
1247err_neigh_init:
1248	return err;
1249}
1250
1251static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv)
1252{
1253	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1254
1255	mlx5e_tc_ht_cleanup(&rpriv->tc_ht);
1256
1257	if (rpriv->rep->vport == MLX5_VPORT_UPLINK)
1258		mlx5e_cleanup_uplink_rep_tx(rpriv);
1259
1260	mlx5e_rep_neigh_cleanup(rpriv);
1261}
1262
1263static void mlx5e_rep_enable(struct mlx5e_priv *priv)
1264{
1265	mlx5e_set_netdev_mtu_boundaries(priv);
1266}
1267
1268static void mlx5e_rep_disable(struct mlx5e_priv *priv)
1269{
1270}
1271
1272static int mlx5e_update_rep_rx(struct mlx5e_priv *priv)
1273{
1274	return 0;
1275}
1276
1277static void mlx5e_rep_stats_update_ndo_stats(struct mlx5e_priv *priv)
1278{
1279	mlx5e_stats_update_ndo_stats(priv);
1280	mlx5e_rep_update_vport_q_counter(priv);
1281}
1282
1283static int mlx5e_rep_event_mpesw(struct mlx5e_priv *priv)
1284{
1285	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1286	struct mlx5_eswitch_rep *rep = rpriv->rep;
1287
1288	if (rep->vport != MLX5_VPORT_UPLINK)
1289		return NOTIFY_DONE;
1290
1291	queue_work(priv->wq, &rpriv->uplink_priv.mpesw_work);
1292
1293	return NOTIFY_OK;
1294}
1295
1296static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data)
1297{
1298	struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
1299
1300	if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
1301		struct mlx5_eqe *eqe = data;
1302
1303		switch (eqe->sub_type) {
1304		case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
1305		case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
1306			queue_work(priv->wq, &priv->update_carrier_work);
1307			break;
1308		default:
1309			return NOTIFY_DONE;
1310		}
1311
1312		return NOTIFY_OK;
1313	}
1314
1315	if (event == MLX5_DEV_EVENT_PORT_AFFINITY)
1316		return mlx5e_rep_tc_event_port_affinity(priv);
1317	else if (event == MLX5_DEV_EVENT_MULTIPORT_ESW)
1318		return mlx5e_rep_event_mpesw(priv);
1319
1320	return NOTIFY_DONE;
1321}
1322
1323static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv)
1324{
1325	struct net_device *netdev = priv->netdev;
1326	struct mlx5_core_dev *mdev = priv->mdev;
1327	u16 max_mtu;
1328
1329	mlx5e_ipsec_init(priv);
1330
1331	netdev->min_mtu = ETH_MIN_MTU;
1332	mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
1333	netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
1334	mlx5e_set_dev_port_mtu(priv);
1335
1336	mlx5e_rep_tc_enable(priv);
1337
1338	if (MLX5_CAP_GEN(mdev, uplink_follow))
1339		mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK,
1340					      0, 0, MLX5_VPORT_ADMIN_STATE_AUTO);
1341	mlx5_lag_add_netdev(mdev, netdev);
1342	priv->events_nb.notifier_call = uplink_rep_async_event;
1343	mlx5_notifier_register(mdev, &priv->events_nb);
1344	mlx5e_dcbnl_initialize(priv);
1345	mlx5e_dcbnl_init_app(priv);
1346	mlx5e_rep_bridge_init(priv);
1347
1348	netdev->wanted_features |= NETIF_F_HW_TC;
1349
1350	rtnl_lock();
1351	netdev_lock(netdev);
1352	if (netif_running(netdev))
1353		mlx5e_open(netdev);
1354	udp_tunnel_nic_reset_ntf(priv->netdev);
1355	netdev_unlock(netdev);
1356	netif_device_attach(netdev);
1357	rtnl_unlock();
1358}
1359
1360static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv)
1361{
1362	struct mlx5_core_dev *mdev = priv->mdev;
1363
1364	rtnl_lock();
1365	netdev_lock(priv->netdev);
1366	if (netif_running(priv->netdev))
1367		mlx5e_close(priv->netdev);
1368	netif_device_detach(priv->netdev);
1369	netdev_unlock(priv->netdev);
1370	rtnl_unlock();
1371
1372	mlx5e_rep_bridge_cleanup(priv);
1373	mlx5e_dcbnl_delete_app(priv);
1374	mlx5_notifier_unregister(mdev, &priv->events_nb);
1375	mlx5e_rep_tc_disable(priv);
1376	mlx5_lag_remove_netdev(mdev, priv->netdev);
1377	mlx5_vxlan_reset_to_default(mdev->vxlan);
1378
1379	mlx5e_ipsec_cleanup(priv);
1380}
1381
1382static MLX5E_DEFINE_STATS_GRP(sw_rep, 0);
1383static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS);
1384
1385/* The stats groups order is opposite to the update_stats() order calls */
1386static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = {
1387	&MLX5E_STATS_GRP(sw_rep),
1388	&MLX5E_STATS_GRP(vport_rep),
1389};
1390
1391static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv)
1392{
1393	return ARRAY_SIZE(mlx5e_rep_stats_grps);
1394}
1395
1396/* The stats groups order is opposite to the update_stats() order calls */
1397static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = {
1398	&MLX5E_STATS_GRP(sw),
1399	&MLX5E_STATS_GRP(qcnt),
1400	&MLX5E_STATS_GRP(vnic_env),
1401	&MLX5E_STATS_GRP(vport),
1402	&MLX5E_STATS_GRP(802_3),
1403	&MLX5E_STATS_GRP(2863),
1404	&MLX5E_STATS_GRP(2819),
1405	&MLX5E_STATS_GRP(phy),
1406	&MLX5E_STATS_GRP(eth_ext),
1407	&MLX5E_STATS_GRP(pcie),
1408	&MLX5E_STATS_GRP(per_prio),
1409	&MLX5E_STATS_GRP(pme),
1410	&MLX5E_STATS_GRP(channels),
1411	&MLX5E_STATS_GRP(per_port_buff_congest),
1412#ifdef CONFIG_MLX5_EN_IPSEC
1413	&MLX5E_STATS_GRP(ipsec_hw),
1414	&MLX5E_STATS_GRP(ipsec_sw),
1415#endif
1416	&MLX5E_STATS_GRP(ptp),
1417};
1418
1419static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv)
1420{
1421	return ARRAY_SIZE(mlx5e_ul_rep_stats_grps);
1422}
1423
1424static int
1425mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter,
1426				 struct devlink_fmsg *fmsg,
1427				 struct netlink_ext_ack *extack)
1428{
1429	struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter);
1430	struct mlx5_eswitch_rep *rep = rpriv->rep;
1431
1432	mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg, rep->vport,
1433					     true);
1434	return 0;
1435}
1436
1437static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = {
1438	.name = "vnic",
1439	.diagnose = mlx5e_rep_vnic_reporter_diagnose,
1440};
1441
1442static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv,
1443					   struct devlink_port *dl_port)
1444{
1445	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1446	struct devlink_health_reporter *reporter;
1447
1448	reporter = devl_port_health_reporter_create(dl_port,
1449						    &mlx5_rep_vnic_reporter_ops,
1450						    rpriv);
1451	if (IS_ERR(reporter)) {
1452		mlx5_core_err(priv->mdev,
1453			      "Failed to create representor vnic reporter, err = %pe\n",
1454			      reporter);
1455		return;
1456	}
1457
1458	rpriv->rep_vnic_reporter = reporter;
1459}
1460
1461static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv)
1462{
1463	struct mlx5e_rep_priv *rpriv = priv->ppriv;
1464
1465	if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter))
1466		devl_health_reporter_destroy(rpriv->rep_vnic_reporter);
1467}
1468
1469static const struct mlx5e_profile mlx5e_rep_profile = {
1470	.init			= mlx5e_init_rep,
1471	.cleanup		= mlx5e_cleanup_rep,
1472	.init_rx		= mlx5e_init_rep_rx,
1473	.cleanup_rx		= mlx5e_cleanup_rep_rx,
1474	.init_tx		= mlx5e_init_rep_tx,
1475	.cleanup_tx		= mlx5e_cleanup_rep_tx,
1476	.enable		        = mlx5e_rep_enable,
1477	.disable	        = mlx5e_rep_disable,
1478	.update_rx		= mlx5e_update_rep_rx,
1479	.update_stats           = mlx5e_rep_stats_update_ndo_stats,
1480	.rx_handlers            = &mlx5e_rx_handlers_rep,
1481	.max_tc			= 1,
1482	.stats_grps		= mlx5e_rep_stats_grps,
1483	.stats_grps_num		= mlx5e_rep_stats_grps_num,
1484	.max_nch_limit		= mlx5e_rep_max_nch_limit,
1485};
1486
1487static const struct mlx5e_profile mlx5e_uplink_rep_profile = {
1488	.init			= mlx5e_init_ul_rep,
1489	.cleanup		= mlx5e_cleanup_rep,
1490	.init_rx		= mlx5e_init_ul_rep_rx,
1491	.cleanup_rx		= mlx5e_cleanup_ul_rep_rx,
1492	.init_tx		= mlx5e_init_rep_tx,
1493	.cleanup_tx		= mlx5e_cleanup_rep_tx,
1494	.enable		        = mlx5e_uplink_rep_enable,
1495	.disable	        = mlx5e_uplink_rep_disable,
1496	.update_rx		= mlx5e_update_rep_rx,
1497	.update_stats           = mlx5e_stats_update_ndo_stats,
1498	.update_carrier	        = mlx5e_update_carrier,
1499	.rx_handlers            = &mlx5e_rx_handlers_rep,
1500	.max_tc			= MLX5_MAX_NUM_TC,
1501	.stats_grps		= mlx5e_ul_rep_stats_grps,
1502	.stats_grps_num		= mlx5e_ul_rep_stats_grps_num,
1503};
1504
1505/* e-Switch vport representors */
1506static int
1507mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1508{
1509	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1510	struct net_device *netdev;
1511	int err;
1512
1513	netdev = mlx5_uplink_netdev_get(dev);
1514	if (!netdev)
1515		return 0;
1516
1517	/* must not use netdev_priv(netdev), it might not be initialized yet */
1518	rpriv->netdev = netdev;
1519	err = mlx5e_netdev_change_profile(netdev, dev,
1520					  &mlx5e_uplink_rep_profile, rpriv);
1521	mlx5_uplink_netdev_put(dev, netdev);
1522	return err;
1523}
1524
1525static void
1526mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv *rpriv)
1527{
1528	struct net_device *netdev = rpriv->netdev;
1529	struct mlx5e_priv *priv;
1530
1531	priv = netdev_priv(netdev);
1532
1533	/* This bit is set when using devlink to change eswitch mode from
1534	 * switchdev to legacy. As need to keep uplink netdev ifindex, we
1535	 * detach uplink representor profile and attach NIC profile only.
1536	 * The netdev will be unregistered later when unload NIC auxiliary
1537	 * driver for this case.
1538	 * We explicitly block devlink eswitch mode change if any IPSec rules
1539	 * offloaded, but can't block other cases, such as driver unload
1540	 * and devlink reload. We have to unregister netdev before profile
1541	 * change for those cases. This is to avoid resource leak because
1542	 * the offloaded rules don't have the chance to be unoffloaded before
1543	 * cleanup which is triggered by detach uplink representor profile.
1544	 */
1545	if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_SWITCH_LEGACY))
1546		unregister_netdev(netdev);
1547
1548	mlx5e_netdev_attach_nic_profile(netdev, priv->mdev);
1549}
1550
1551static int
1552mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1553{
1554	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1555	const struct mlx5e_profile *profile;
1556	struct devlink_port *dl_port;
1557	struct net_device *netdev;
1558	struct mlx5e_priv *priv;
1559	int err;
1560
1561	profile = &mlx5e_rep_profile;
1562	netdev = mlx5e_create_netdev(dev, profile);
1563	if (!netdev) {
1564		mlx5_core_warn(dev,
1565			       "Failed to create representor netdev for vport %d\n",
1566			       rep->vport);
1567		return -EINVAL;
1568	}
1569
1570	mlx5e_build_rep_netdev(netdev, dev);
1571	rpriv->netdev = netdev;
1572
1573	priv = netdev_priv(netdev);
1574	priv->profile = profile;
1575	priv->ppriv = rpriv;
1576	err = profile->init(dev, netdev);
1577	if (err) {
1578		netdev_warn(netdev, "rep profile init failed, %d\n", err);
1579		goto err_destroy_netdev;
1580	}
1581
1582	err = mlx5e_attach_netdev(netdev_priv(netdev));
1583	if (err) {
1584		netdev_warn(netdev,
1585			    "Failed to attach representor netdev for vport %d\n",
1586			    rep->vport);
1587		goto err_cleanup_profile;
1588	}
1589
1590	dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch,
1591						 rpriv->rep->vport);
1592	if (!IS_ERR(dl_port)) {
1593		SET_NETDEV_DEVLINK_PORT(netdev, dl_port);
1594		mlx5e_rep_vnic_reporter_create(priv, dl_port);
1595	}
1596
1597	err = register_netdev(netdev);
1598	if (err) {
1599		netdev_warn(netdev,
1600			    "Failed to register representor netdev for vport %d\n",
1601			    rep->vport);
1602		goto err_detach_netdev;
1603	}
1604
1605	return 0;
1606
1607err_detach_netdev:
1608	mlx5e_rep_vnic_reporter_destroy(priv);
1609	mlx5e_detach_netdev(netdev_priv(netdev));
1610err_cleanup_profile:
1611	priv->profile->cleanup(priv);
1612
1613err_destroy_netdev:
1614	mlx5e_destroy_netdev(netdev);
1615	return err;
1616}
1617
1618static int
1619mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1620{
1621	struct mlx5e_rep_priv *rpriv;
1622	int err;
1623
1624	rpriv = kvzalloc(sizeof(*rpriv), GFP_KERNEL);
1625	if (!rpriv)
1626		return -ENOMEM;
1627
1628	/* rpriv->rep to be looked up when profile->init() is called */
1629	rpriv->rep = rep;
1630	rep->rep_data[REP_ETH].priv = rpriv;
1631	INIT_LIST_HEAD(&rpriv->vport_sqs_list);
1632
1633	if (rep->vport == MLX5_VPORT_UPLINK)
1634		err = mlx5e_vport_uplink_rep_load(dev, rep);
1635	else
1636		err = mlx5e_vport_vf_rep_load(dev, rep);
1637
1638	if (err)
1639		kvfree(rpriv);
1640
1641	return err;
1642}
1643
1644static void
1645mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
1646{
1647	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
1648	struct net_device *netdev = rpriv->netdev;
1649	struct mlx5e_priv *priv;
1650	void *ppriv;
1651
1652	if (!netdev) {
1653		ppriv = rpriv;
1654		goto free_ppriv;
1655	}
1656
1657	priv = netdev_priv(netdev);
1658	ppriv = priv->ppriv;
1659
1660	if (rep->vport == MLX5_VPORT_UPLINK) {
1661		mlx5e_vport_uplink_rep_unload(rpriv);
1662		goto free_ppriv;
1663	}
1664
1665	unregister_netdev(netdev);
1666	mlx5e_rep_vnic_reporter_destroy(priv);
1667	mlx5e_detach_netdev(priv);
1668	priv->profile->cleanup(priv);
1669	mlx5e_destroy_netdev(netdev);
1670free_ppriv:
1671	kvfree(ppriv); /* mlx5e_rep_priv */
1672}
1673
1674static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep)
1675{
1676	struct mlx5e_rep_priv *rpriv;
1677
1678	rpriv = mlx5e_rep_to_rep_priv(rep);
1679
1680	return rpriv->netdev;
1681}
1682
1683static void mlx5e_vport_rep_event_unpair(struct mlx5_eswitch_rep *rep,
1684					 struct mlx5_eswitch *peer_esw)
1685{
1686	u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
1687	struct mlx5e_rep_priv *rpriv;
1688	struct mlx5e_rep_sq *rep_sq;
1689
1690	WARN_ON_ONCE(!peer_esw);
1691	rpriv = mlx5e_rep_to_rep_priv(rep);
1692	list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) {
1693		struct mlx5e_rep_sq_peer *sq_peer = xa_load(&rep_sq->sq_peer, i);
1694
1695		if (!sq_peer || sq_peer->peer != peer_esw)
1696			continue;
1697
1698		mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule);
1699		xa_erase(&rep_sq->sq_peer, i);
1700		kfree(sq_peer);
1701	}
1702}
1703
1704static int mlx5e_vport_rep_event_pair(struct mlx5_eswitch *esw,
1705				      struct mlx5_eswitch_rep *rep,
1706				      struct mlx5_eswitch *peer_esw)
1707{
1708	u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id);
1709	struct mlx5_flow_handle *flow_rule;
1710	struct mlx5e_rep_sq_peer *sq_peer;
1711	struct mlx5e_rep_priv *rpriv;
1712	struct mlx5e_rep_sq *rep_sq;
1713	int err;
1714
1715	rpriv = mlx5e_rep_to_rep_priv(rep);
1716	list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) {
1717		sq_peer = xa_load(&rep_sq->sq_peer, i);
1718
1719		if (sq_peer && sq_peer->peer)
1720			continue;
1721
1722		flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw, rep,
1723								rep_sq->sqn);
1724		if (IS_ERR(flow_rule)) {
1725			err = PTR_ERR(flow_rule);
1726			goto err_out;
1727		}
1728
1729		if (sq_peer) {
1730			sq_peer->rule = flow_rule;
1731			sq_peer->peer = peer_esw;
1732			continue;
1733		}
1734		sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL);
1735		if (!sq_peer) {
1736			err = -ENOMEM;
1737			goto err_sq_alloc;
1738		}
1739		err = xa_insert(&rep_sq->sq_peer, i, sq_peer, GFP_KERNEL);
1740		if (err)
1741			goto err_xa;
1742		sq_peer->rule = flow_rule;
1743		sq_peer->peer = peer_esw;
1744	}
1745
1746	return 0;
1747err_xa:
1748	kfree(sq_peer);
1749err_sq_alloc:
1750	mlx5_eswitch_del_send_to_vport_rule(flow_rule);
1751err_out:
1752	mlx5e_vport_rep_event_unpair(rep, peer_esw);
1753	return err;
1754}
1755
1756static int mlx5e_vport_rep_event(struct mlx5_eswitch *esw,
1757				 struct mlx5_eswitch_rep *rep,
1758				 enum mlx5_switchdev_event event,
1759				 void *data)
1760{
1761	int err = 0;
1762
1763	if (event == MLX5_SWITCHDEV_EVENT_PAIR)
1764		err = mlx5e_vport_rep_event_pair(esw, rep, data);
1765	else if (event == MLX5_SWITCHDEV_EVENT_UNPAIR)
1766		mlx5e_vport_rep_event_unpair(rep, data);
1767
1768	return err;
1769}
1770
1771static const struct mlx5_eswitch_rep_ops rep_ops = {
1772	.load = mlx5e_vport_rep_load,
1773	.unload = mlx5e_vport_rep_unload,
1774	.get_proto_dev = mlx5e_vport_rep_get_proto_dev,
1775	.event = mlx5e_vport_rep_event,
1776};
1777
1778static int mlx5e_rep_probe(struct auxiliary_device *adev,
1779			   const struct auxiliary_device_id *id)
1780{
1781	struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
1782	struct mlx5_core_dev *mdev = edev->mdev;
1783	struct mlx5_eswitch *esw;
1784
1785	esw = mdev->priv.eswitch;
1786	mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH);
1787	return 0;
1788}
1789
1790static void mlx5e_rep_remove(struct auxiliary_device *adev)
1791{
1792	struct mlx5_adev *vdev = container_of(adev, struct mlx5_adev, adev);
1793	struct mlx5_core_dev *mdev = vdev->mdev;
1794	struct mlx5_eswitch *esw;
1795
1796	esw = mdev->priv.eswitch;
1797	mlx5_eswitch_unregister_vport_reps(esw, REP_ETH);
1798}
1799
1800static const struct auxiliary_device_id mlx5e_rep_id_table[] = {
1801	{ .name = MLX5_ADEV_NAME ".eth-rep", },
1802	{},
1803};
1804
1805MODULE_DEVICE_TABLE(auxiliary, mlx5e_rep_id_table);
1806
1807static struct auxiliary_driver mlx5e_rep_driver = {
1808	.name = "eth-rep",
1809	.probe = mlx5e_rep_probe,
1810	.remove = mlx5e_rep_remove,
1811	.id_table = mlx5e_rep_id_table,
1812};
1813
1814int mlx5e_rep_init(void)
1815{
1816	return auxiliary_driver_register(&mlx5e_rep_driver);
1817}
1818
1819void mlx5e_rep_cleanup(void)
1820{
1821	auxiliary_driver_unregister(&mlx5e_rep_driver);
1822}