ena_ethtool.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
   4 */
   5
   6#include <linux/ethtool.h>
   7#include <linux/pci.h>
   8#include <linux/net_tstamp.h>
   9
  10#include "ena_netdev.h"
  11#include "ena_xdp.h"
  12#include "ena_phc.h"
  13
  14struct ena_stats {
  15	char name[ETH_GSTRING_LEN];
  16	int stat_offset;
  17};
  18
  19struct ena_hw_metrics {
  20	char name[ETH_GSTRING_LEN];
  21};
  22
  23#define ENA_STAT_ENA_COM_ENTRY(stat) { \
  24	.name = #stat, \
  25	.stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \
  26}
  27
  28#define ENA_STAT_ENTRY(stat, stat_type) { \
  29	.name = #stat, \
  30	.stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \
  31}
  32
  33#define ENA_STAT_HW_ENTRY(stat, stat_type) { \
  34	.name = #stat, \
  35	.stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \
  36}
  37
  38#define ENA_STAT_RX_ENTRY(stat) \
  39	ENA_STAT_ENTRY(stat, rx)
  40
  41#define ENA_STAT_TX_ENTRY(stat) \
  42	ENA_STAT_ENTRY(stat, tx)
  43
  44#define ENA_STAT_GLOBAL_ENTRY(stat) \
  45	ENA_STAT_ENTRY(stat, dev)
  46
  47#define ENA_STAT_ENI_ENTRY(stat) \
  48	ENA_STAT_HW_ENTRY(stat, eni_stats)
  49
  50#define ENA_STAT_ENA_SRD_ENTRY(stat) \
  51	ENA_STAT_HW_ENTRY(stat, ena_srd_stats)
  52
  53#define ENA_STAT_ENA_SRD_MODE_ENTRY(stat) { \
  54	.name = #stat, \
  55	.stat_offset = offsetof(struct ena_admin_ena_srd_info, flags) / sizeof(u64) \
  56}
  57
  58#define ENA_METRIC_ENI_ENTRY(stat) { \
  59	.name = #stat \
  60}
  61
  62static const struct ena_stats ena_stats_global_strings[] = {
  63	ENA_STAT_GLOBAL_ENTRY(tx_timeout),
  64	ENA_STAT_GLOBAL_ENTRY(suspend),
  65	ENA_STAT_GLOBAL_ENTRY(resume),
  66	ENA_STAT_GLOBAL_ENTRY(wd_expired),
  67	ENA_STAT_GLOBAL_ENTRY(interface_up),
  68	ENA_STAT_GLOBAL_ENTRY(interface_down),
  69	ENA_STAT_GLOBAL_ENTRY(admin_q_pause),
  70	ENA_STAT_GLOBAL_ENTRY(reset_fail),
  71};
  72
  73/* A partial list of hw stats. Used when admin command
  74 * with type ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS is not supported
  75 */
  76static const struct ena_stats ena_stats_eni_strings[] = {
  77	ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded),
  78	ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded),
  79	ENA_STAT_ENI_ENTRY(pps_allowance_exceeded),
  80	ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded),
  81	ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded),
  82};
  83
  84static const struct ena_hw_metrics ena_hw_stats_strings[] = {
  85	ENA_METRIC_ENI_ENTRY(bw_in_allowance_exceeded),
  86	ENA_METRIC_ENI_ENTRY(bw_out_allowance_exceeded),
  87	ENA_METRIC_ENI_ENTRY(pps_allowance_exceeded),
  88	ENA_METRIC_ENI_ENTRY(conntrack_allowance_exceeded),
  89	ENA_METRIC_ENI_ENTRY(linklocal_allowance_exceeded),
  90	ENA_METRIC_ENI_ENTRY(conntrack_allowance_available),
  91};
  92
  93static const struct ena_stats ena_srd_info_strings[] = {
  94	ENA_STAT_ENA_SRD_MODE_ENTRY(ena_srd_mode),
  95	ENA_STAT_ENA_SRD_ENTRY(ena_srd_tx_pkts),
  96	ENA_STAT_ENA_SRD_ENTRY(ena_srd_eligible_tx_pkts),
  97	ENA_STAT_ENA_SRD_ENTRY(ena_srd_rx_pkts),
  98	ENA_STAT_ENA_SRD_ENTRY(ena_srd_resource_utilization)
  99};
 100
 101static const struct ena_stats ena_stats_tx_strings[] = {
 102	ENA_STAT_TX_ENTRY(cnt),
 103	ENA_STAT_TX_ENTRY(bytes),
 104	ENA_STAT_TX_ENTRY(queue_stop),
 105	ENA_STAT_TX_ENTRY(queue_wakeup),
 106	ENA_STAT_TX_ENTRY(dma_mapping_err),
 107	ENA_STAT_TX_ENTRY(linearize),
 108	ENA_STAT_TX_ENTRY(linearize_failed),
 109	ENA_STAT_TX_ENTRY(napi_comp),
 110	ENA_STAT_TX_ENTRY(tx_poll),
 111	ENA_STAT_TX_ENTRY(doorbells),
 112	ENA_STAT_TX_ENTRY(prepare_ctx_err),
 113	ENA_STAT_TX_ENTRY(bad_req_id),
 114	ENA_STAT_TX_ENTRY(llq_buffer_copy),
 115	ENA_STAT_TX_ENTRY(missed_tx),
 116	ENA_STAT_TX_ENTRY(unmask_interrupt),
 117};
 118
 119static const struct ena_stats ena_stats_rx_strings[] = {
 120	ENA_STAT_RX_ENTRY(cnt),
 121	ENA_STAT_RX_ENTRY(bytes),
 122	ENA_STAT_RX_ENTRY(rx_copybreak_pkt),
 123	ENA_STAT_RX_ENTRY(csum_good),
 124	ENA_STAT_RX_ENTRY(refil_partial),
 125	ENA_STAT_RX_ENTRY(csum_bad),
 126	ENA_STAT_RX_ENTRY(page_alloc_fail),
 127	ENA_STAT_RX_ENTRY(skb_alloc_fail),
 128	ENA_STAT_RX_ENTRY(dma_mapping_err),
 129	ENA_STAT_RX_ENTRY(bad_desc_num),
 130	ENA_STAT_RX_ENTRY(bad_req_id),
 131	ENA_STAT_RX_ENTRY(empty_rx_ring),
 132	ENA_STAT_RX_ENTRY(csum_unchecked),
 133	ENA_STAT_RX_ENTRY(xdp_aborted),
 134	ENA_STAT_RX_ENTRY(xdp_drop),
 135	ENA_STAT_RX_ENTRY(xdp_pass),
 136	ENA_STAT_RX_ENTRY(xdp_tx),
 137	ENA_STAT_RX_ENTRY(xdp_invalid),
 138	ENA_STAT_RX_ENTRY(xdp_redirect),
 139};
 140
 141static const struct ena_stats ena_stats_ena_com_strings[] = {
 142	ENA_STAT_ENA_COM_ENTRY(aborted_cmd),
 143	ENA_STAT_ENA_COM_ENTRY(submitted_cmd),
 144	ENA_STAT_ENA_COM_ENTRY(completed_cmd),
 145	ENA_STAT_ENA_COM_ENTRY(out_of_space),
 146	ENA_STAT_ENA_COM_ENTRY(no_completion),
 147};
 148
 149#define ENA_STATS_ARRAY_GLOBAL		ARRAY_SIZE(ena_stats_global_strings)
 150#define ENA_STATS_ARRAY_TX		ARRAY_SIZE(ena_stats_tx_strings)
 151#define ENA_STATS_ARRAY_RX		ARRAY_SIZE(ena_stats_rx_strings)
 152#define ENA_STATS_ARRAY_ENA_COM		ARRAY_SIZE(ena_stats_ena_com_strings)
 153#define ENA_STATS_ARRAY_ENI		ARRAY_SIZE(ena_stats_eni_strings)
 154#define ENA_STATS_ARRAY_ENA_SRD		ARRAY_SIZE(ena_srd_info_strings)
 155#define ENA_METRICS_ARRAY_ENI		ARRAY_SIZE(ena_hw_stats_strings)
 156
 157static void ena_safe_update_stat(u64 *src, u64 *dst,
 158				 struct u64_stats_sync *syncp)
 159{
 160	unsigned int start;
 161
 162	do {
 163		start = u64_stats_fetch_begin(syncp);
 164		*(dst) = *src;
 165	} while (u64_stats_fetch_retry(syncp, start));
 166}
 167
 168static void ena_metrics_stats(struct ena_adapter *adapter, u64 **data)
 169{
 170	struct ena_com_dev *dev = adapter->ena_dev;
 171	const struct ena_stats *ena_stats;
 172	u64 *ptr;
 173	int i;
 174
 175	if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) {
 176		u32 supported_metrics_count;
 177		int len;
 178
 179		supported_metrics_count = ena_com_get_customer_metric_count(dev);
 180		len = supported_metrics_count * sizeof(u64);
 181
 182		/* Fill the data buffer, and advance its pointer */
 183		ena_com_get_customer_metrics(dev, (char *)(*data), len);
 184		(*data) += supported_metrics_count;
 185
 186	} else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) {
 187		ena_com_get_eni_stats(dev, &adapter->eni_stats);
 188		/* Updating regardless of rc - once we told ethtool how many stats we have
 189		 * it will print that much stats. We can't leave holes in the stats
 190		 */
 191		for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) {
 192			ena_stats = &ena_stats_eni_strings[i];
 193
 194			ptr = (u64 *)&adapter->eni_stats +
 195				ena_stats->stat_offset;
 196
 197			ena_safe_update_stat(ptr, (*data)++, &adapter->syncp);
 198		}
 199	}
 200
 201	if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) {
 202		ena_com_get_ena_srd_info(dev, &adapter->ena_srd_info);
 203		/* Get ENA SRD mode */
 204		ptr = (u64 *)&adapter->ena_srd_info;
 205		ena_safe_update_stat(ptr, (*data)++, &adapter->syncp);
 206		for (i = 1; i < ENA_STATS_ARRAY_ENA_SRD; i++) {
 207			ena_stats = &ena_srd_info_strings[i];
 208			/* Wrapped within an outer struct - need to accommodate an
 209			 * additional offset of the ENA SRD mode that was already processed
 210			 */
 211			ptr = (u64 *)&adapter->ena_srd_info +
 212				ena_stats->stat_offset + 1;
 213
 214			ena_safe_update_stat(ptr, (*data)++, &adapter->syncp);
 215		}
 216	}
 217}
 218
 219static void ena_queue_stats(struct ena_adapter *adapter, u64 **data)
 220{
 221	const struct ena_stats *ena_stats;
 222	struct ena_ring *ring;
 223
 224	u64 *ptr;
 225	int i, j;
 226
 227	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 228		/* Tx stats */
 229		ring = &adapter->tx_ring[i];
 230
 231		for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 232			ena_stats = &ena_stats_tx_strings[j];
 233
 234			ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset;
 235
 236			ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 237		}
 238		/* XDP TX queues don't have a RX queue counterpart */
 239		if (!ENA_IS_XDP_INDEX(adapter, i)) {
 240			/* Rx stats */
 241			ring = &adapter->rx_ring[i];
 242
 243			for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 244				ena_stats = &ena_stats_rx_strings[j];
 245
 246				ptr = (u64 *)&ring->rx_stats +
 247					ena_stats->stat_offset;
 248
 249				ena_safe_update_stat(ptr, (*data)++, &ring->syncp);
 250			}
 251		}
 252	}
 253}
 254
 255static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data)
 256{
 257	const struct ena_stats *ena_stats;
 258	u64 *ptr;
 259	int i;
 260
 261	for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 262		ena_stats = &ena_stats_ena_com_strings[i];
 263
 264		ptr = (u64 *)&adapter->ena_dev->admin_queue.stats +
 265			ena_stats->stat_offset;
 266
 267		*(*data)++ = *ptr;
 268	}
 269}
 270
 271static void ena_get_stats(struct ena_adapter *adapter,
 272			  u64 *data,
 273			  bool hw_stats_needed)
 274{
 275	const struct ena_stats *ena_stats;
 276	u64 *ptr;
 277	int i;
 278
 279	for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 280		ena_stats = &ena_stats_global_strings[i];
 281
 282		ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset;
 283
 284		ena_safe_update_stat(ptr, data++, &adapter->syncp);
 285	}
 286
 287	if (hw_stats_needed)
 288		ena_metrics_stats(adapter, &data);
 289
 290	ena_queue_stats(adapter, &data);
 291	ena_dev_admin_queue_stats(adapter, &data);
 292}
 293
 294static void ena_get_ethtool_stats(struct net_device *netdev,
 295				  struct ethtool_stats *stats,
 296				  u64 *data)
 297{
 298	struct ena_adapter *adapter = netdev_priv(netdev);
 299
 300	ena_get_stats(adapter, data, true);
 301}
 302
 303static int ena_get_ts_info(struct net_device *netdev,
 304			   struct kernel_ethtool_ts_info *info)
 305{
 306	struct ena_adapter *adapter = netdev_priv(netdev);
 307
 308	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE;
 309
 310	info->phc_index = ena_phc_get_index(adapter);
 311
 312	return 0;
 313}
 314
 315static int ena_get_sw_stats_count(struct ena_adapter *adapter)
 316{
 317	return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX)
 318		+ adapter->xdp_num_queues * ENA_STATS_ARRAY_TX
 319		+ ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM;
 320}
 321
 322static int ena_get_hw_stats_count(struct ena_adapter *adapter)
 323{
 324	struct ena_com_dev *dev = adapter->ena_dev;
 325	int count;
 326
 327	count = ENA_STATS_ARRAY_ENA_SRD * ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO);
 328
 329	if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS))
 330		count += ena_com_get_customer_metric_count(dev);
 331	else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS))
 332		count += ENA_STATS_ARRAY_ENI;
 333
 334	return count;
 335}
 336
 337int ena_get_sset_count(struct net_device *netdev, int sset)
 338{
 339	struct ena_adapter *adapter = netdev_priv(netdev);
 340
 341	switch (sset) {
 342	case ETH_SS_STATS:
 343		return ena_get_sw_stats_count(adapter) +
 344		       ena_get_hw_stats_count(adapter);
 345	}
 346
 347	return -EOPNOTSUPP;
 348}
 349
 350static void ena_metrics_stats_strings(struct ena_adapter *adapter, u8 **data)
 351{
 352	struct ena_com_dev *dev = adapter->ena_dev;
 353	const struct ena_hw_metrics *ena_metrics;
 354	const struct ena_stats *ena_stats;
 355	int i;
 356
 357	if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) {
 358		for (i = 0; i < ENA_METRICS_ARRAY_ENI; i++) {
 359			if (ena_com_get_customer_metric_support(dev, i)) {
 360				ena_metrics = &ena_hw_stats_strings[i];
 361				ethtool_puts(data, ena_metrics->name);
 362			}
 363		}
 364	} else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) {
 365		for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) {
 366			ena_stats = &ena_stats_eni_strings[i];
 367			ethtool_puts(data, ena_stats->name);
 368		}
 369	}
 370
 371	if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) {
 372		for (i = 0; i < ENA_STATS_ARRAY_ENA_SRD; i++) {
 373			ena_stats = &ena_srd_info_strings[i];
 374			ethtool_puts(data, ena_stats->name);
 375		}
 376	}
 377}
 378
 379static void ena_queue_strings(struct ena_adapter *adapter, u8 **data)
 380{
 381	const struct ena_stats *ena_stats;
 382	bool is_xdp;
 383	int i, j;
 384
 385	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
 386		is_xdp = ENA_IS_XDP_INDEX(adapter, i);
 387		/* Tx stats */
 388		for (j = 0; j < ENA_STATS_ARRAY_TX; j++) {
 389			ena_stats = &ena_stats_tx_strings[j];
 390
 391			ethtool_sprintf(data,
 392					"queue_%u_%s_%s", i,
 393					is_xdp ? "xdp_tx" : "tx",
 394					ena_stats->name);
 395		}
 396
 397		/* In XDP there isn't an RX queue counterpart */
 398		if (is_xdp)
 399			continue;
 400
 401		for (j = 0; j < ENA_STATS_ARRAY_RX; j++) {
 402			ena_stats = &ena_stats_rx_strings[j];
 403
 404			ethtool_sprintf(data, "queue_%u_rx_%s", i, ena_stats->name);
 405		}
 406	}
 407}
 408
 409static void ena_com_dev_strings(u8 **data)
 410{
 411	const struct ena_stats *ena_stats;
 412	int i;
 413
 414	for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) {
 415		ena_stats = &ena_stats_ena_com_strings[i];
 416
 417		ethtool_sprintf(data,
 418				"ena_admin_q_%s", ena_stats->name);
 419	}
 420}
 421
 422static void ena_get_strings(struct ena_adapter *adapter,
 423			    u8 *data,
 424			    bool hw_stats_needed)
 425{
 426	const struct ena_stats *ena_stats;
 427	int i;
 428
 429	for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) {
 430		ena_stats = &ena_stats_global_strings[i];
 431		ethtool_puts(&data, ena_stats->name);
 432	}
 433
 434	if (hw_stats_needed)
 435		ena_metrics_stats_strings(adapter, &data);
 436
 437	ena_queue_strings(adapter, &data);
 438	ena_com_dev_strings(&data);
 439}
 440
 441static void ena_get_ethtool_strings(struct net_device *netdev,
 442				    u32 sset,
 443				    u8 *data)
 444{
 445	struct ena_adapter *adapter = netdev_priv(netdev);
 446
 447	switch (sset) {
 448	case ETH_SS_STATS:
 449		ena_get_strings(adapter, data, true);
 450		break;
 451	}
 452}
 453
 454static int ena_get_link_ksettings(struct net_device *netdev,
 455				  struct ethtool_link_ksettings *link_ksettings)
 456{
 457	struct ena_adapter *adapter = netdev_priv(netdev);
 458	struct ena_com_dev *ena_dev = adapter->ena_dev;
 459	struct ena_admin_get_feature_link_desc *link;
 460	struct ena_admin_get_feat_resp feat_resp;
 461	int rc;
 462
 463	rc = ena_com_get_link_params(ena_dev, &feat_resp);
 464	if (rc)
 465		return rc;
 466
 467	link = &feat_resp.u.link;
 468	link_ksettings->base.speed = link->speed;
 469
 470	if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) {
 471		ethtool_link_ksettings_add_link_mode(link_ksettings,
 472						     supported, Autoneg);
 473		ethtool_link_ksettings_add_link_mode(link_ksettings,
 474						     supported, Autoneg);
 475	}
 476
 477	link_ksettings->base.autoneg =
 478		(link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ?
 479		AUTONEG_ENABLE : AUTONEG_DISABLE;
 480
 481	link_ksettings->base.duplex = DUPLEX_FULL;
 482
 483	return 0;
 484}
 485
 486static int ena_get_coalesce(struct net_device *net_dev,
 487			    struct ethtool_coalesce *coalesce,
 488			    struct kernel_ethtool_coalesce *kernel_coal,
 489			    struct netlink_ext_ack *extack)
 490{
 491	struct ena_adapter *adapter = netdev_priv(net_dev);
 492	struct ena_com_dev *ena_dev = adapter->ena_dev;
 493
 494	if (!ena_com_interrupt_moderation_supported(ena_dev))
 495		return -EOPNOTSUPP;
 496
 497	coalesce->tx_coalesce_usecs =
 498		ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) *
 499			ena_dev->intr_delay_resolution;
 500
 501	coalesce->rx_coalesce_usecs =
 502		ena_com_get_nonadaptive_moderation_interval_rx(ena_dev)
 503		* ena_dev->intr_delay_resolution;
 504
 505	coalesce->use_adaptive_rx_coalesce =
 506		ena_com_get_adaptive_moderation_enabled(ena_dev);
 507
 508	return 0;
 509}
 510
 511static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 512{
 513	unsigned int val;
 514	int i;
 515
 516	val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev);
 517
 518	for (i = 0; i < adapter->num_io_queues; i++)
 519		adapter->tx_ring[i].smoothed_interval = val;
 520}
 521
 522static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter)
 523{
 524	unsigned int val;
 525	int i;
 526
 527	val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev);
 528
 529	for (i = 0; i < adapter->num_io_queues; i++)
 530		adapter->rx_ring[i].smoothed_interval = val;
 531}
 532
 533static int ena_set_coalesce(struct net_device *net_dev,
 534			    struct ethtool_coalesce *coalesce,
 535			    struct kernel_ethtool_coalesce *kernel_coal,
 536			    struct netlink_ext_ack *extack)
 537{
 538	struct ena_adapter *adapter = netdev_priv(net_dev);
 539	struct ena_com_dev *ena_dev = adapter->ena_dev;
 540	int rc;
 541
 542	if (!ena_com_interrupt_moderation_supported(ena_dev))
 543		return -EOPNOTSUPP;
 544
 545	rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev,
 546							       coalesce->tx_coalesce_usecs);
 547	if (rc)
 548		return rc;
 549
 550	ena_update_tx_rings_nonadaptive_intr_moderation(adapter);
 551
 552	rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev,
 553							       coalesce->rx_coalesce_usecs);
 554	if (rc)
 555		return rc;
 556
 557	ena_update_rx_rings_nonadaptive_intr_moderation(adapter);
 558
 559	if (coalesce->use_adaptive_rx_coalesce &&
 560	    !ena_com_get_adaptive_moderation_enabled(ena_dev))
 561		ena_com_enable_adaptive_moderation(ena_dev);
 562
 563	if (!coalesce->use_adaptive_rx_coalesce &&
 564	    ena_com_get_adaptive_moderation_enabled(ena_dev))
 565		ena_com_disable_adaptive_moderation(ena_dev);
 566
 567	return 0;
 568}
 569
 570static u32 ena_get_msglevel(struct net_device *netdev)
 571{
 572	struct ena_adapter *adapter = netdev_priv(netdev);
 573
 574	return adapter->msg_enable;
 575}
 576
 577static void ena_set_msglevel(struct net_device *netdev, u32 value)
 578{
 579	struct ena_adapter *adapter = netdev_priv(netdev);
 580
 581	adapter->msg_enable = value;
 582}
 583
 584static void ena_get_drvinfo(struct net_device *dev,
 585			    struct ethtool_drvinfo *info)
 586{
 587	struct ena_adapter *adapter = netdev_priv(dev);
 588	ssize_t ret = 0;
 589
 590	ret = strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 591	if (ret < 0)
 592		netif_dbg(adapter, drv, dev,
 593			  "module name will be truncated, status = %zd\n", ret);
 594
 595	ret = strscpy(info->bus_info, pci_name(adapter->pdev),
 596		      sizeof(info->bus_info));
 597	if (ret < 0)
 598		netif_dbg(adapter, drv, dev,
 599			  "bus info will be truncated, status = %zd\n", ret);
 600}
 601
 602static void ena_get_ringparam(struct net_device *netdev,
 603			      struct ethtool_ringparam *ring,
 604			      struct kernel_ethtool_ringparam *kernel_ring,
 605			      struct netlink_ext_ack *extack)
 606{
 607	struct ena_adapter *adapter = netdev_priv(netdev);
 608
 609	ring->tx_max_pending = adapter->max_tx_ring_size;
 610	ring->rx_max_pending = adapter->max_rx_ring_size;
 611	if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
 612		bool large_llq_supported = adapter->large_llq_header_supported;
 613
 614		kernel_ring->tx_push = true;
 615		kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 616		if (large_llq_supported)
 617			kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER;
 618		else
 619			kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER;
 620	} else {
 621		kernel_ring->tx_push = false;
 622		kernel_ring->tx_push_buf_max_len = 0;
 623		kernel_ring->tx_push_buf_len = 0;
 624	}
 625
 626	ring->tx_pending = adapter->tx_ring[0].ring_size;
 627	ring->rx_pending = adapter->rx_ring[0].ring_size;
 628}
 629
 630static int ena_set_ringparam(struct net_device *netdev,
 631			     struct ethtool_ringparam *ring,
 632			     struct kernel_ethtool_ringparam *kernel_ring,
 633			     struct netlink_ext_ack *extack)
 634{
 635	struct ena_adapter *adapter = netdev_priv(netdev);
 636	u32 new_tx_size, new_rx_size, new_tx_push_buf_len;
 637	bool changed = false;
 638
 639	new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ?
 640			ENA_MIN_RING_SIZE : ring->tx_pending;
 641	new_tx_size = rounddown_pow_of_two(new_tx_size);
 642
 643	new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ?
 644			ENA_MIN_RING_SIZE : ring->rx_pending;
 645	new_rx_size = rounddown_pow_of_two(new_rx_size);
 646
 647	changed |= new_tx_size != adapter->requested_tx_ring_size ||
 648		   new_rx_size != adapter->requested_rx_ring_size;
 649
 650	/* This value is ignored if LLQ is not supported */
 651	new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size;
 652
 653	if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) !=
 654	    kernel_ring->tx_push) {
 655		NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified");
 656		return -EINVAL;
 657	}
 658
 659	/* Validate that the push buffer is supported on the underlying device */
 660	if (kernel_ring->tx_push_buf_len) {
 661		enum ena_admin_placement_policy_type placement;
 662
 663		new_tx_push_buf_len = kernel_ring->tx_push_buf_len;
 664
 665		placement = adapter->ena_dev->tx_mem_queue_type;
 666		if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST)
 667			return -EOPNOTSUPP;
 668
 669		if (new_tx_push_buf_len != ENA_LLQ_HEADER &&
 670		    new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) {
 671			bool large_llq_sup = adapter->large_llq_header_supported;
 672			char large_llq_size_str[40];
 673
 674			snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER);
 675
 676			NL_SET_ERR_MSG_FMT_MOD(extack,
 677					       "Supported tx push buff values: [%lu%s]",
 678					       ENA_LLQ_HEADER,
 679					       large_llq_sup ? large_llq_size_str : "");
 680
 681			return -EINVAL;
 682		}
 683
 684		changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size;
 685	}
 686
 687	if (!changed)
 688		return 0;
 689
 690	return ena_update_queue_params(adapter, new_tx_size, new_rx_size,
 691				       new_tx_push_buf_len);
 692}
 693
 694static u32 ena_flow_hash_to_flow_type(u16 hash_fields)
 695{
 696	u32 data = 0;
 697
 698	if (hash_fields & ENA_ADMIN_RSS_L2_DA)
 699		data |= RXH_L2DA;
 700
 701	if (hash_fields & ENA_ADMIN_RSS_L3_DA)
 702		data |= RXH_IP_DST;
 703
 704	if (hash_fields & ENA_ADMIN_RSS_L3_SA)
 705		data |= RXH_IP_SRC;
 706
 707	if (hash_fields & ENA_ADMIN_RSS_L4_DP)
 708		data |= RXH_L4_B_2_3;
 709
 710	if (hash_fields & ENA_ADMIN_RSS_L4_SP)
 711		data |= RXH_L4_B_0_1;
 712
 713	return data;
 714}
 715
 716static u16 ena_flow_data_to_flow_hash(u32 hash_fields)
 717{
 718	u16 data = 0;
 719
 720	if (hash_fields & RXH_L2DA)
 721		data |= ENA_ADMIN_RSS_L2_DA;
 722
 723	if (hash_fields & RXH_IP_DST)
 724		data |= ENA_ADMIN_RSS_L3_DA;
 725
 726	if (hash_fields & RXH_IP_SRC)
 727		data |= ENA_ADMIN_RSS_L3_SA;
 728
 729	if (hash_fields & RXH_L4_B_2_3)
 730		data |= ENA_ADMIN_RSS_L4_DP;
 731
 732	if (hash_fields & RXH_L4_B_0_1)
 733		data |= ENA_ADMIN_RSS_L4_SP;
 734
 735	return data;
 736}
 737
 738static int ena_get_rxfh_fields(struct net_device *netdev,
 739			       struct ethtool_rxfh_fields *cmd)
 740{
 741	struct ena_adapter *adapter = netdev_priv(netdev);
 742	struct ena_com_dev *ena_dev = adapter->ena_dev;
 743	enum ena_admin_flow_hash_proto proto;
 744	u16 hash_fields;
 745	int rc;
 746
 747	cmd->data = 0;
 748
 749	switch (cmd->flow_type) {
 750	case TCP_V4_FLOW:
 751		proto = ENA_ADMIN_RSS_TCP4;
 752		break;
 753	case UDP_V4_FLOW:
 754		proto = ENA_ADMIN_RSS_UDP4;
 755		break;
 756	case TCP_V6_FLOW:
 757		proto = ENA_ADMIN_RSS_TCP6;
 758		break;
 759	case UDP_V6_FLOW:
 760		proto = ENA_ADMIN_RSS_UDP6;
 761		break;
 762	case IPV4_FLOW:
 763		proto = ENA_ADMIN_RSS_IP4;
 764		break;
 765	case IPV6_FLOW:
 766		proto = ENA_ADMIN_RSS_IP6;
 767		break;
 768	case ETHER_FLOW:
 769		proto = ENA_ADMIN_RSS_NOT_IP;
 770		break;
 771	case AH_V4_FLOW:
 772	case ESP_V4_FLOW:
 773	case AH_V6_FLOW:
 774	case ESP_V6_FLOW:
 775	case SCTP_V4_FLOW:
 776	case AH_ESP_V4_FLOW:
 777		return -EOPNOTSUPP;
 778	default:
 779		return -EINVAL;
 780	}
 781
 782	rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields);
 783	if (rc)
 784		return rc;
 785
 786	cmd->data = ena_flow_hash_to_flow_type(hash_fields);
 787
 788	return 0;
 789}
 790
 791static int ena_set_rxfh_fields(struct net_device *netdev,
 792			       const struct ethtool_rxfh_fields *cmd,
 793			       struct netlink_ext_ack *extack)
 794{
 795	struct ena_adapter *adapter = netdev_priv(netdev);
 796	struct ena_com_dev *ena_dev = adapter->ena_dev;
 797	enum ena_admin_flow_hash_proto proto;
 798	u16 hash_fields;
 799
 800	switch (cmd->flow_type) {
 801	case TCP_V4_FLOW:
 802		proto = ENA_ADMIN_RSS_TCP4;
 803		break;
 804	case UDP_V4_FLOW:
 805		proto = ENA_ADMIN_RSS_UDP4;
 806		break;
 807	case TCP_V6_FLOW:
 808		proto = ENA_ADMIN_RSS_TCP6;
 809		break;
 810	case UDP_V6_FLOW:
 811		proto = ENA_ADMIN_RSS_UDP6;
 812		break;
 813	case IPV4_FLOW:
 814		proto = ENA_ADMIN_RSS_IP4;
 815		break;
 816	case IPV6_FLOW:
 817		proto = ENA_ADMIN_RSS_IP6;
 818		break;
 819	case ETHER_FLOW:
 820		proto = ENA_ADMIN_RSS_NOT_IP;
 821		break;
 822	case AH_V4_FLOW:
 823	case ESP_V4_FLOW:
 824	case AH_V6_FLOW:
 825	case ESP_V6_FLOW:
 826	case SCTP_V4_FLOW:
 827	case AH_ESP_V4_FLOW:
 828		return -EOPNOTSUPP;
 829	default:
 830		return -EINVAL;
 831	}
 832
 833	hash_fields = ena_flow_data_to_flow_hash(cmd->data);
 834
 835	return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields);
 836}
 837
 838static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
 839			 u32 *rules)
 840{
 841	struct ena_adapter *adapter = netdev_priv(netdev);
 842	int rc = 0;
 843
 844	switch (info->cmd) {
 845	case ETHTOOL_GRXRINGS:
 846		info->data = adapter->num_io_queues;
 847		rc = 0;
 848		break;
 849	case ETHTOOL_GRXCLSRLCNT:
 850	case ETHTOOL_GRXCLSRULE:
 851	case ETHTOOL_GRXCLSRLALL:
 852	default:
 853		netif_err(adapter, drv, netdev,
 854			  "Command parameter %d is not supported\n", info->cmd);
 855		rc = -EOPNOTSUPP;
 856	}
 857
 858	return rc;
 859}
 860
 861static u32 ena_get_rxfh_indir_size(struct net_device *netdev)
 862{
 863	return ENA_RX_RSS_TABLE_SIZE;
 864}
 865
 866static u32 ena_get_rxfh_key_size(struct net_device *netdev)
 867{
 868	struct ena_adapter *adapter = netdev_priv(netdev);
 869	struct ena_rss *rss = &adapter->ena_dev->rss;
 870
 871	return rss->hash_key ? ENA_HASH_KEY_SIZE : 0;
 872}
 873
 874static int ena_indirection_table_set(struct ena_adapter *adapter,
 875				     const u32 *indir)
 876{
 877	struct ena_com_dev *ena_dev = adapter->ena_dev;
 878	int i, rc;
 879
 880	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
 881		rc = ena_com_indirect_table_fill_entry(ena_dev,
 882						       i,
 883						       ENA_IO_RXQ_IDX(indir[i]));
 884		if (unlikely(rc)) {
 885			netif_err(adapter, drv, adapter->netdev,
 886				  "Cannot fill indirect table (index is too large)\n");
 887			return rc;
 888		}
 889	}
 890
 891	rc = ena_com_indirect_table_set(ena_dev);
 892	if (rc) {
 893		netif_err(adapter, drv, adapter->netdev,
 894			  "Cannot set indirect table\n");
 895		return rc == -EPERM ? -EOPNOTSUPP : rc;
 896	}
 897	return rc;
 898}
 899
 900static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir)
 901{
 902	struct ena_com_dev *ena_dev = adapter->ena_dev;
 903	int i, rc;
 904
 905	if (!indir)
 906		return 0;
 907
 908	rc = ena_com_indirect_table_get(ena_dev, indir);
 909	if (rc)
 910		return rc;
 911
 912	/* Our internal representation of the indices is: even indices
 913	 * for Tx and uneven indices for Rx. We need to convert the Rx
 914	 * indices to be consecutive
 915	 */
 916	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++)
 917		indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]);
 918
 919	return rc;
 920}
 921
 922static int ena_get_rxfh(struct net_device *netdev,
 923			struct ethtool_rxfh_param *rxfh)
 924{
 925	struct ena_adapter *adapter = netdev_priv(netdev);
 926	enum ena_admin_hash_functions ena_func;
 927	u8 func;
 928	int rc;
 929
 930	rc = ena_indirection_table_get(adapter, rxfh->indir);
 931	if (rc)
 932		return rc;
 933
 934	/* We call this function in order to check if the device
 935	 * supports getting/setting the hash function.
 936	 */
 937	rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func);
 938	if (rc) {
 939		if (rc == -EOPNOTSUPP)
 940			rc = 0;
 941
 942		return rc;
 943	}
 944
 945	rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key);
 946	if (rc)
 947		return rc;
 948
 949	switch (ena_func) {
 950	case ENA_ADMIN_TOEPLITZ:
 951		func = ETH_RSS_HASH_TOP;
 952		break;
 953	case ENA_ADMIN_CRC32:
 954		func = ETH_RSS_HASH_CRC32;
 955		break;
 956	default:
 957		netif_err(adapter, drv, netdev,
 958			  "Command parameter is not supported\n");
 959		return -EOPNOTSUPP;
 960	}
 961
 962	rxfh->hfunc = func;
 963
 964	return 0;
 965}
 966
 967static int ena_set_rxfh(struct net_device *netdev,
 968			struct ethtool_rxfh_param *rxfh,
 969			struct netlink_ext_ack *extack)
 970{
 971	struct ena_adapter *adapter = netdev_priv(netdev);
 972	struct ena_com_dev *ena_dev = adapter->ena_dev;
 973	enum ena_admin_hash_functions func = 0;
 974	int rc;
 975
 976	if (rxfh->indir) {
 977		rc = ena_indirection_table_set(adapter, rxfh->indir);
 978		if (rc)
 979			return rc;
 980	}
 981
 982	switch (rxfh->hfunc) {
 983	case ETH_RSS_HASH_NO_CHANGE:
 984		func = ena_com_get_current_hash_function(ena_dev);
 985		break;
 986	case ETH_RSS_HASH_TOP:
 987		func = ENA_ADMIN_TOEPLITZ;
 988		break;
 989	case ETH_RSS_HASH_CRC32:
 990		func = ENA_ADMIN_CRC32;
 991		break;
 992	default:
 993		netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n",
 994			  rxfh->hfunc);
 995		return -EOPNOTSUPP;
 996	}
 997
 998	if (rxfh->key || func) {
 999		rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key,
1000						ENA_HASH_KEY_SIZE,
1001						0xFFFFFFFF);
1002		if (unlikely(rc)) {
1003			netif_err(adapter, drv, netdev, "Cannot fill key\n");
1004			return rc == -EPERM ? -EOPNOTSUPP : rc;
1005		}
1006	}
1007
1008	return 0;
1009}
1010
1011static void ena_get_channels(struct net_device *netdev,
1012			     struct ethtool_channels *channels)
1013{
1014	struct ena_adapter *adapter = netdev_priv(netdev);
1015
1016	channels->max_combined = adapter->max_num_io_queues;
1017	channels->combined_count = adapter->num_io_queues;
1018}
1019
1020static int ena_set_channels(struct net_device *netdev,
1021			    struct ethtool_channels *channels)
1022{
1023	struct ena_adapter *adapter = netdev_priv(netdev);
1024	u32 count = channels->combined_count;
1025	/* The check for max value is already done in ethtool */
1026	if (count < ENA_MIN_NUM_IO_QUEUES)
1027		return -EINVAL;
1028
1029	if (!ena_xdp_legal_queue_count(adapter, count)) {
1030		if (ena_xdp_present(adapter))
1031			return -EINVAL;
1032
1033		xdp_clear_features_flag(netdev);
1034	} else {
1035		xdp_set_features_flag(netdev,
1036				      NETDEV_XDP_ACT_BASIC |
1037				      NETDEV_XDP_ACT_REDIRECT);
1038	}
1039
1040	return ena_update_queue_count(adapter, count);
1041}
1042
1043static int ena_get_tunable(struct net_device *netdev,
1044			   const struct ethtool_tunable *tuna, void *data)
1045{
1046	struct ena_adapter *adapter = netdev_priv(netdev);
1047	int ret = 0;
1048
1049	switch (tuna->id) {
1050	case ETHTOOL_RX_COPYBREAK:
1051		*(u32 *)data = adapter->rx_copybreak;
1052		break;
1053	default:
1054		ret = -EINVAL;
1055		break;
1056	}
1057
1058	return ret;
1059}
1060
1061static int ena_set_tunable(struct net_device *netdev,
1062			   const struct ethtool_tunable *tuna,
1063			   const void *data)
1064{
1065	struct ena_adapter *adapter = netdev_priv(netdev);
1066	int ret = 0;
1067	u32 len;
1068
1069	switch (tuna->id) {
1070	case ETHTOOL_RX_COPYBREAK:
1071		len = *(u32 *)data;
1072		ret = ena_set_rx_copybreak(adapter, len);
1073		break;
1074	default:
1075		ret = -EINVAL;
1076		break;
1077	}
1078
1079	return ret;
1080}
1081
1082static const struct ethtool_ops ena_ethtool_ops = {
1083	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
1084				     ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
1085	.supported_ring_params	= ETHTOOL_RING_USE_TX_PUSH_BUF_LEN |
1086				  ETHTOOL_RING_USE_TX_PUSH,
1087	.get_link_ksettings	= ena_get_link_ksettings,
1088	.get_drvinfo		= ena_get_drvinfo,
1089	.get_msglevel		= ena_get_msglevel,
1090	.set_msglevel		= ena_set_msglevel,
1091	.get_link		= ethtool_op_get_link,
1092	.get_coalesce		= ena_get_coalesce,
1093	.set_coalesce		= ena_set_coalesce,
1094	.get_ringparam		= ena_get_ringparam,
1095	.set_ringparam		= ena_set_ringparam,
1096	.get_sset_count         = ena_get_sset_count,
1097	.get_strings		= ena_get_ethtool_strings,
1098	.get_ethtool_stats      = ena_get_ethtool_stats,
1099	.get_rxnfc		= ena_get_rxnfc,
1100	.get_rxfh_indir_size    = ena_get_rxfh_indir_size,
1101	.get_rxfh_key_size	= ena_get_rxfh_key_size,
1102	.get_rxfh		= ena_get_rxfh,
1103	.set_rxfh		= ena_set_rxfh,
1104	.get_rxfh_fields	= ena_get_rxfh_fields,
1105	.set_rxfh_fields	= ena_set_rxfh_fields,
1106	.get_channels		= ena_get_channels,
1107	.set_channels		= ena_set_channels,
1108	.get_tunable		= ena_get_tunable,
1109	.set_tunable		= ena_set_tunable,
1110	.get_ts_info		= ena_get_ts_info,
1111};
1112
1113void ena_set_ethtool_ops(struct net_device *netdev)
1114{
1115	netdev->ethtool_ops = &ena_ethtool_ops;
1116}
1117
1118static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf)
1119{
1120	struct net_device *netdev = adapter->netdev;
1121	u8 *strings_buf;
1122	u64 *data_buf;
1123	int strings_num;
1124	int i, rc;
1125
1126	strings_num = ena_get_sw_stats_count(adapter);
1127	if (strings_num <= 0) {
1128		netif_err(adapter, drv, netdev, "Can't get stats num\n");
1129		return;
1130	}
1131
1132	strings_buf = kcalloc(strings_num, ETH_GSTRING_LEN, GFP_ATOMIC);
1133	if (!strings_buf) {
1134		netif_err(adapter, drv, netdev,
1135			  "Failed to allocate strings_buf\n");
1136		return;
1137	}
1138
1139	data_buf = kcalloc(strings_num, sizeof(u64), GFP_ATOMIC);
1140	if (!data_buf) {
1141		netif_err(adapter, drv, netdev,
1142			  "Failed to allocate data buf\n");
1143		kfree(strings_buf);
1144		return;
1145	}
1146
1147	ena_get_strings(adapter, strings_buf, false);
1148	ena_get_stats(adapter, data_buf, false);
1149
1150	/* If there is a buffer, dump stats, otherwise print them to dmesg */
1151	if (buf)
1152		for (i = 0; i < strings_num; i++) {
1153			rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64),
1154				      "%s %llu\n",
1155				      strings_buf + i * ETH_GSTRING_LEN,
1156				      data_buf[i]);
1157			buf += rc;
1158		}
1159	else
1160		for (i = 0; i < strings_num; i++)
1161			netif_err(adapter, drv, netdev, "%s: %llu\n",
1162				  strings_buf + i * ETH_GSTRING_LEN,
1163				  data_buf[i]);
1164
1165	kfree(strings_buf);
1166	kfree(data_buf);
1167}
1168
1169void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf)
1170{
1171	if (!buf)
1172		return;
1173
1174	ena_dump_stats_ex(adapter, buf);
1175}
1176
1177void ena_dump_stats_to_dmesg(struct ena_adapter *adapter)
1178{
1179	ena_dump_stats_ex(adapter, NULL);
1180}