FILE
ice_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/intel/ice/ice_main.c
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2018-2023, Intel Corporation. */ 3 4/* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8#include <generated/utsrelease.h> 9#include <linux/crash_dump.h> 10#include "ice.h" 11#include "ice_base.h" 12#include "ice_lib.h" 13#include "ice_fltr.h" 14#include "ice_dcb_lib.h" 15#include "ice_dcb_nl.h" 16#include "devlink/devlink.h" 17#include "devlink/port.h" 18#include "ice_sf_eth.h" 19#include "ice_hwmon.h" 20/* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the 21 * ice tracepoint functions. This must be done exactly once across the 22 * ice driver. 23 */ 24#define CREATE_TRACE_POINTS 25#include "ice_trace.h" 26#include "ice_eswitch.h" 27#include "ice_tc_lib.h" 28#include "ice_vsi_vlan_ops.h" 29#include <net/xdp_sock_drv.h> 30 31#define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 32static const char ice_driver_string[] = DRV_SUMMARY; 33static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 34 35/* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 36#define ICE_DDP_PKG_PATH "intel/ice/ddp/" 37#define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 38 39MODULE_DESCRIPTION(DRV_SUMMARY); 40MODULE_IMPORT_NS("LIBETH"); 41MODULE_IMPORT_NS("LIBETH_XDP"); 42MODULE_IMPORT_NS("LIBIE"); 43MODULE_IMPORT_NS("LIBIE_ADMINQ"); 44MODULE_IMPORT_NS("LIBIE_FWLOG"); 45MODULE_LICENSE("GPL v2"); 46MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 47 48static int debug = -1; 49module_param(debug, int, 0644); 50#ifndef CONFIG_DYNAMIC_DEBUG 51MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 52#else 53MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 54#endif /* !CONFIG_DYNAMIC_DEBUG */ 55 56DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key); 57EXPORT_SYMBOL(ice_xdp_locking_key); 58 59/** 60 * ice_hw_to_dev - Get device pointer from the hardware structure 61 * @hw: pointer to the device HW structure 62 * 63 * Used to access the device pointer from compilation units which can't easily 64 * include the definition of struct ice_pf without leading to circular header 65 * dependencies. 66 */ 67struct device *ice_hw_to_dev(struct ice_hw *hw) 68{ 69 struct ice_pf *pf = container_of(hw, struct ice_pf, hw); 70 71 return &pf->pdev->dev; 72} 73 74static struct workqueue_struct *ice_wq; 75struct workqueue_struct *ice_lag_wq; 76static const struct net_device_ops ice_netdev_safe_mode_ops; 77static const struct net_device_ops ice_netdev_ops; 78 79static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 80 81static void ice_vsi_release_all(struct ice_pf *pf); 82 83static int ice_rebuild_channels(struct ice_pf *pf); 84static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr); 85 86static int 87ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 88 void *cb_priv, enum tc_setup_type type, void *type_data, 89 void *data, 90 void (*cleanup)(struct flow_block_cb *block_cb)); 91 92bool netif_is_ice(const struct net_device *dev) 93{ 94 return dev && (dev->netdev_ops == &ice_netdev_ops || 95 dev->netdev_ops == &ice_netdev_safe_mode_ops); 96} 97 98/** 99 * ice_get_tx_pending - returns number of Tx descriptors not processed 100 * @ring: the ring of descriptors 101 */ 102static u16 ice_get_tx_pending(struct ice_tx_ring *ring) 103{ 104 u16 head, tail; 105 106 head = ring->next_to_clean; 107 tail = ring->next_to_use; 108 109 if (head != tail) 110 return (head < tail) ? 111 tail - head : (tail + ring->count - head); 112 return 0; 113} 114 115/** 116 * ice_check_for_hang_subtask - check for and recover hung queues 117 * @pf: pointer to PF struct 118 */ 119static void ice_check_for_hang_subtask(struct ice_pf *pf) 120{ 121 struct ice_vsi *vsi = NULL; 122 struct ice_hw *hw; 123 unsigned int i; 124 int packets; 125 u32 v; 126 127 ice_for_each_vsi(pf, v) 128 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 129 vsi = pf->vsi[v]; 130 break; 131 } 132 133 if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state)) 134 return; 135 136 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 137 return; 138 139 hw = &vsi->back->hw; 140 141 ice_for_each_txq(vsi, i) { 142 struct ice_tx_ring *tx_ring = vsi->tx_rings[i]; 143 struct ice_ring_stats *ring_stats; 144 145 if (!tx_ring) 146 continue; 147 if (ice_ring_ch_enabled(tx_ring)) 148 continue; 149 150 ring_stats = tx_ring->ring_stats; 151 if (!ring_stats) 152 continue; 153 154 if (tx_ring->desc) { 155 /* If packet counter has not changed the queue is 156 * likely stalled, so force an interrupt for this 157 * queue. 158 * 159 * prev_pkt would be negative if there was no 160 * pending work. 161 */ 162 packets = ring_stats->stats.pkts & INT_MAX; 163 if (ring_stats->tx_stats.prev_pkt == packets) { 164 /* Trigger sw interrupt to revive the queue */ 165 ice_trigger_sw_intr(hw, tx_ring->q_vector); 166 continue; 167 } 168 169 /* Memory barrier between read of packet count and call 170 * to ice_get_tx_pending() 171 */ 172 smp_rmb(); 173 ring_stats->tx_stats.prev_pkt = 174 ice_get_tx_pending(tx_ring) ? packets : -1; 175 } 176 } 177} 178 179/** 180 * ice_init_mac_fltr - Set initial MAC filters 181 * @pf: board private structure 182 * 183 * Set initial set of MAC filters for PF VSI; configure filters for permanent 184 * address and broadcast address. If an error is encountered, netdevice will be 185 * unregistered. 186 */ 187static int ice_init_mac_fltr(struct ice_pf *pf) 188{ 189 struct ice_vsi *vsi; 190 u8 *perm_addr; 191 192 vsi = ice_get_main_vsi(pf); 193 if (!vsi) 194 return -EINVAL; 195 196 perm_addr = vsi->port_info->mac.perm_addr; 197 return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 198} 199 200/** 201 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 202 * @netdev: the net device on which the sync is happening 203 * @addr: MAC address to sync 204 * 205 * This is a callback function which is called by the in kernel device sync 206 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 207 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 208 * MAC filters from the hardware. 209 */ 210static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 211{ 212 struct ice_netdev_priv *np = netdev_priv(netdev); 213 struct ice_vsi *vsi = np->vsi; 214 215 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 216 ICE_FWD_TO_VSI)) 217 return -EINVAL; 218 219 return 0; 220} 221 222/** 223 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 224 * @netdev: the net device on which the unsync is happening 225 * @addr: MAC address to unsync 226 * 227 * This is a callback function which is called by the in kernel device unsync 228 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 229 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 230 * delete the MAC filters from the hardware. 231 */ 232static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 233{ 234 struct ice_netdev_priv *np = netdev_priv(netdev); 235 struct ice_vsi *vsi = np->vsi; 236 237 /* Under some circumstances, we might receive a request to delete our 238 * own device address from our uc list. Because we store the device 239 * address in the VSI's MAC filter list, we need to ignore such 240 * requests and not delete our device address from this list. 241 */ 242 if (ether_addr_equal(addr, netdev->dev_addr)) 243 return 0; 244 245 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 246 ICE_FWD_TO_VSI)) 247 return -EINVAL; 248 249 return 0; 250} 251 252/** 253 * ice_vsi_fltr_changed - check if filter state changed 254 * @vsi: VSI to be checked 255 * 256 * returns true if filter state has changed, false otherwise. 257 */ 258static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 259{ 260 return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) || 261 test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 262} 263 264/** 265 * ice_set_promisc - Enable promiscuous mode for a given PF 266 * @vsi: the VSI being configured 267 * @promisc_m: mask of promiscuous config bits 268 * 269 */ 270static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m) 271{ 272 int status; 273 274 if (vsi->type != ICE_VSI_PF) 275 return 0; 276 277 if (ice_vsi_has_non_zero_vlans(vsi)) { 278 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 279 status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi, 280 promisc_m); 281 } else { 282 status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 283 promisc_m, 0); 284 } 285 if (status && status != -EEXIST) 286 return status; 287 288 netdev_dbg(vsi->netdev, "set promisc filter bits for VSI %i: 0x%x\n", 289 vsi->vsi_num, promisc_m); 290 return 0; 291} 292 293/** 294 * ice_clear_promisc - Disable promiscuous mode for a given PF 295 * @vsi: the VSI being configured 296 * @promisc_m: mask of promiscuous config bits 297 * 298 */ 299static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m) 300{ 301 int status; 302 303 if (vsi->type != ICE_VSI_PF) 304 return 0; 305 306 if (ice_vsi_has_non_zero_vlans(vsi)) { 307 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 308 status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi, 309 promisc_m); 310 } else { 311 status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 312 promisc_m, 0); 313 } 314 315 netdev_dbg(vsi->netdev, "clear promisc filter bits for VSI %i: 0x%x\n", 316 vsi->vsi_num, promisc_m); 317 return status; 318} 319 320/** 321 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 322 * @vsi: ptr to the VSI 323 * 324 * Push any outstanding VSI filter changes through the AdminQ. 325 */ 326static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 327{ 328 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 329 struct device *dev = ice_pf_to_dev(vsi->back); 330 struct net_device *netdev = vsi->netdev; 331 bool promisc_forced_on = false; 332 struct ice_pf *pf = vsi->back; 333 struct ice_hw *hw = &pf->hw; 334 u32 changed_flags = 0; 335 int err; 336 337 if (!vsi->netdev) 338 return -EINVAL; 339 340 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 341 usleep_range(1000, 2000); 342 343 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 344 vsi->current_netdev_flags = vsi->netdev->flags; 345 346 INIT_LIST_HEAD(&vsi->tmp_sync_list); 347 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 348 349 if (ice_vsi_fltr_changed(vsi)) { 350 clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 351 clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 352 353 /* grab the netdev's addr_list_lock */ 354 netif_addr_lock_bh(netdev); 355 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 356 ice_add_mac_to_unsync_list); 357 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 358 ice_add_mac_to_unsync_list); 359 /* our temp lists are populated. release lock */ 360 netif_addr_unlock_bh(netdev); 361 } 362 363 /* Remove MAC addresses in the unsync list */ 364 err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 365 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 366 if (err) { 367 netdev_err(netdev, "Failed to delete MAC filters\n"); 368 /* if we failed because of alloc failures, just bail */ 369 if (err == -ENOMEM) 370 goto out; 371 } 372 373 /* Add MAC addresses in the sync list */ 374 err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 375 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 376 /* If filter is added successfully or already exists, do not go into 377 * 'if' condition and report it as error. Instead continue processing 378 * rest of the function. 379 */ 380 if (err && err != -EEXIST) { 381 netdev_err(netdev, "Failed to add MAC filters\n"); 382 /* If there is no more space for new umac filters, VSI 383 * should go into promiscuous mode. There should be some 384 * space reserved for promiscuous filters. 385 */ 386 if (hw->adminq.sq_last_status == LIBIE_AQ_RC_ENOSPC && 387 !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC, 388 vsi->state)) { 389 promisc_forced_on = true; 390 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 391 vsi->vsi_num); 392 } else { 393 goto out; 394 } 395 } 396 err = 0; 397 /* check for changes in promiscuous modes */ 398 if (changed_flags & IFF_ALLMULTI) { 399 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 400 err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS); 401 if (err) { 402 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 403 goto out_promisc; 404 } 405 } else { 406 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 407 err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS); 408 if (err) { 409 vsi->current_netdev_flags |= IFF_ALLMULTI; 410 goto out_promisc; 411 } 412 } 413 } 414 415 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 416 test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) { 417 clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 418 if (vsi->current_netdev_flags & IFF_PROMISC) { 419 /* Apply Rx filter rule to get traffic from wire */ 420 if (!ice_is_dflt_vsi_in_use(vsi->port_info)) { 421 err = ice_set_dflt_vsi(vsi); 422 if (err && err != -EEXIST) { 423 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 424 err, vsi->vsi_num); 425 vsi->current_netdev_flags &= 426 ~IFF_PROMISC; 427 goto out_promisc; 428 } 429 err = 0; 430 vlan_ops->dis_rx_filtering(vsi); 431 432 /* promiscuous mode implies allmulticast so 433 * that VSIs that are in promiscuous mode are 434 * subscribed to multicast packets coming to 435 * the port 436 */ 437 err = ice_set_promisc(vsi, 438 ICE_MCAST_PROMISC_BITS); 439 if (err) 440 goto out_promisc; 441 } 442 } else { 443 /* Clear Rx filter to remove traffic from wire */ 444 if (ice_is_vsi_dflt_vsi(vsi)) { 445 err = ice_clear_dflt_vsi(vsi); 446 if (err) { 447 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 448 err, vsi->vsi_num); 449 vsi->current_netdev_flags |= 450 IFF_PROMISC; 451 goto out_promisc; 452 } 453 if (vsi->netdev->features & 454 NETIF_F_HW_VLAN_CTAG_FILTER) 455 vlan_ops->ena_rx_filtering(vsi); 456 } 457 458 /* disable allmulti here, but only if allmulti is not 459 * still enabled for the netdev 460 */ 461 if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) { 462 err = ice_clear_promisc(vsi, 463 ICE_MCAST_PROMISC_BITS); 464 if (err) { 465 netdev_err(netdev, "Error %d clearing multicast promiscuous on VSI %i\n", 466 err, vsi->vsi_num); 467 } 468 } 469 } 470 } 471 goto exit; 472 473out_promisc: 474 set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 475 goto exit; 476out: 477 /* if something went wrong then set the changed flag so we try again */ 478 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 479 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 480exit: 481 clear_bit(ICE_CFG_BUSY, vsi->state); 482 return err; 483} 484 485/** 486 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 487 * @pf: board private structure 488 */ 489static void ice_sync_fltr_subtask(struct ice_pf *pf) 490{ 491 int v; 492 493 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 494 return; 495 496 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 497 498 ice_for_each_vsi(pf, v) 499 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 500 ice_vsi_sync_fltr(pf->vsi[v])) { 501 /* come back and try again later */ 502 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 503 break; 504 } 505} 506 507/** 508 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 509 * @pf: the PF 510 * @locked: is the rtnl_lock already held 511 */ 512static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 513{ 514 int node; 515 int v; 516 517 ice_for_each_vsi(pf, v) 518 if (pf->vsi[v]) 519 ice_dis_vsi(pf->vsi[v], locked); 520 521 for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++) 522 pf->pf_agg_node[node].num_vsis = 0; 523 524 for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++) 525 pf->vf_agg_node[node].num_vsis = 0; 526} 527 528/** 529 * ice_prepare_for_reset - prep for reset 530 * @pf: board private structure 531 * @reset_type: reset type requested 532 * 533 * Inform or close all dependent features in prep for reset. 534 */ 535static void 536ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 537{ 538 struct ice_hw *hw = &pf->hw; 539 struct ice_vsi *vsi; 540 struct ice_vf *vf; 541 unsigned int bkt; 542 543 dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type); 544 545 /* already prepared for reset */ 546 if (test_bit(ICE_PREPARED_FOR_RESET, pf->state)) 547 return; 548 549 synchronize_irq(pf->oicr_irq.virq); 550 551 ice_unplug_aux_dev(pf); 552 553 /* Notify VFs of impending reset */ 554 if (ice_check_sq_alive(hw, &hw->mailboxq)) 555 ice_vc_notify_reset(pf); 556 557 /* Disable VFs until reset is completed */ 558 mutex_lock(&pf->vfs.table_lock); 559 ice_for_each_vf(pf, bkt, vf) 560 ice_set_vf_state_dis(vf); 561 mutex_unlock(&pf->vfs.table_lock); 562 563 if (ice_is_eswitch_mode_switchdev(pf)) { 564 rtnl_lock(); 565 ice_eswitch_br_fdb_flush(pf->eswitch.br_offloads->bridge); 566 rtnl_unlock(); 567 } 568 569 /* release ADQ specific HW and SW resources */ 570 vsi = ice_get_main_vsi(pf); 571 if (!vsi) 572 goto skip; 573 574 /* to be on safe side, reset orig_rss_size so that normal flow 575 * of deciding rss_size can take precedence 576 */ 577 vsi->orig_rss_size = 0; 578 579 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 580 if (reset_type == ICE_RESET_PFR) { 581 vsi->old_ena_tc = vsi->all_enatc; 582 vsi->old_numtc = vsi->all_numtc; 583 } else { 584 ice_remove_q_channels(vsi, true); 585 586 /* for other reset type, do not support channel rebuild 587 * hence reset needed info 588 */ 589 vsi->old_ena_tc = 0; 590 vsi->all_enatc = 0; 591 vsi->old_numtc = 0; 592 vsi->all_numtc = 0; 593 vsi->req_txq = 0; 594 vsi->req_rxq = 0; 595 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 596 memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt)); 597 } 598 } 599 600 if (vsi->netdev) 601 netif_device_detach(vsi->netdev); 602skip: 603 604 /* clear SW filtering DB */ 605 ice_clear_hw_tbls(hw); 606 /* disable the VSIs and their queues that are not already DOWN */ 607 set_bit(ICE_VSI_REBUILD_PENDING, ice_get_main_vsi(pf)->state); 608 ice_pf_dis_all_vsi(pf, false); 609 610 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 611 ice_ptp_prepare_for_reset(pf, reset_type); 612 613 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 614 ice_gnss_exit(pf); 615 616 if (hw->port_info) 617 ice_sched_clear_port(hw->port_info); 618 619 ice_shutdown_all_ctrlq(hw, false); 620 621 set_bit(ICE_PREPARED_FOR_RESET, pf->state); 622} 623 624/** 625 * ice_do_reset - Initiate one of many types of resets 626 * @pf: board private structure 627 * @reset_type: reset type requested before this function was called. 628 */ 629static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 630{ 631 struct device *dev = ice_pf_to_dev(pf); 632 struct ice_hw *hw = &pf->hw; 633 634 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 635 636 if (pf->lag && pf->lag->bonded && reset_type == ICE_RESET_PFR) { 637 dev_dbg(dev, "PFR on a bonded interface, promoting to CORER\n"); 638 reset_type = ICE_RESET_CORER; 639 } 640 641 ice_prepare_for_reset(pf, reset_type); 642 643 /* trigger the reset */ 644 if (ice_reset(hw, reset_type)) { 645 dev_err(dev, "reset %d failed\n", reset_type); 646 set_bit(ICE_RESET_FAILED, pf->state); 647 clear_bit(ICE_RESET_OICR_RECV, pf->state); 648 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 649 clear_bit(ICE_PFR_REQ, pf->state); 650 clear_bit(ICE_CORER_REQ, pf->state); 651 clear_bit(ICE_GLOBR_REQ, pf->state); 652 wake_up(&pf->reset_wait_queue); 653 return; 654 } 655 656 /* PFR is a bit of a special case because it doesn't result in an OICR 657 * interrupt. So for PFR, rebuild after the reset and clear the reset- 658 * associated state bits. 659 */ 660 if (reset_type == ICE_RESET_PFR) { 661 pf->pfr_count++; 662 ice_rebuild(pf, reset_type); 663 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 664 clear_bit(ICE_PFR_REQ, pf->state); 665 wake_up(&pf->reset_wait_queue); 666 ice_reset_all_vfs(pf); 667 } 668} 669 670/** 671 * ice_reset_subtask - Set up for resetting the device and driver 672 * @pf: board private structure 673 */ 674static void ice_reset_subtask(struct ice_pf *pf) 675{ 676 enum ice_reset_req reset_type = ICE_RESET_INVAL; 677 678 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 679 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 680 * of reset is pending and sets bits in pf->state indicating the reset 681 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set 682 * prepare for pending reset if not already (for PF software-initiated 683 * global resets the software should already be prepared for it as 684 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated 685 * by firmware or software on other PFs, that bit is not set so prepare 686 * for the reset now), poll for reset done, rebuild and return. 687 */ 688 if (test_bit(ICE_RESET_OICR_RECV, pf->state)) { 689 /* Perform the largest reset requested */ 690 if (test_and_clear_bit(ICE_CORER_RECV, pf->state)) 691 reset_type = ICE_RESET_CORER; 692 if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state)) 693 reset_type = ICE_RESET_GLOBR; 694 if (test_and_clear_bit(ICE_EMPR_RECV, pf->state)) 695 reset_type = ICE_RESET_EMPR; 696 /* return if no valid reset type requested */ 697 if (reset_type == ICE_RESET_INVAL) 698 return; 699 ice_prepare_for_reset(pf, reset_type); 700 701 /* make sure we are ready to rebuild */ 702 if (ice_check_reset(&pf->hw)) { 703 set_bit(ICE_RESET_FAILED, pf->state); 704 } else { 705 /* done with reset. start rebuild */ 706 pf->hw.reset_ongoing = false; 707 ice_rebuild(pf, reset_type); 708 /* clear bit to resume normal operations, but 709 * ICE_NEEDS_RESTART bit is set in case rebuild failed 710 */ 711 clear_bit(ICE_RESET_OICR_RECV, pf->state); 712 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 713 clear_bit(ICE_PFR_REQ, pf->state); 714 clear_bit(ICE_CORER_REQ, pf->state); 715 clear_bit(ICE_GLOBR_REQ, pf->state); 716 wake_up(&pf->reset_wait_queue); 717 ice_reset_all_vfs(pf); 718 } 719 720 return; 721 } 722 723 /* No pending resets to finish processing. Check for new resets */ 724 if (test_bit(ICE_PFR_REQ, pf->state)) { 725 reset_type = ICE_RESET_PFR; 726 if (pf->lag && pf->lag->bonded) { 727 dev_dbg(ice_pf_to_dev(pf), "PFR on a bonded interface, promoting to CORER\n"); 728 reset_type = ICE_RESET_CORER; 729 } 730 } 731 if (test_bit(ICE_CORER_REQ, pf->state)) 732 reset_type = ICE_RESET_CORER; 733 if (test_bit(ICE_GLOBR_REQ, pf->state)) 734 reset_type = ICE_RESET_GLOBR; 735 /* If no valid reset type requested just return */ 736 if (reset_type == ICE_RESET_INVAL) 737 return; 738 739 /* reset if not already down or busy */ 740 if (!test_bit(ICE_DOWN, pf->state) && 741 !test_bit(ICE_CFG_BUSY, pf->state)) { 742 ice_do_reset(pf, reset_type); 743 } 744} 745 746/** 747 * ice_print_topo_conflict - print topology conflict message 748 * @vsi: the VSI whose topology status is being checked 749 */ 750static void ice_print_topo_conflict(struct ice_vsi *vsi) 751{ 752 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 753 case ICE_AQ_LINK_TOPO_CONFLICT: 754 case ICE_AQ_LINK_MEDIA_CONFLICT: 755 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 756 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 757 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 758 netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n"); 759 break; 760 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 761 if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags)) 762 netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n"); 763 else 764 netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 765 break; 766 default: 767 break; 768 } 769} 770 771/** 772 * ice_print_link_msg - print link up or down message 773 * @vsi: the VSI whose link status is being queried 774 * @isup: boolean for if the link is now up or down 775 */ 776void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 777{ 778 struct ice_aqc_get_phy_caps_data *caps; 779 const char *an_advertised; 780 const char *fec_req; 781 const char *speed; 782 const char *fec; 783 const char *fc; 784 const char *an; 785 int status; 786 787 if (!vsi) 788 return; 789 790 if (vsi->current_isup == isup) 791 return; 792 793 vsi->current_isup = isup; 794 795 if (!isup) { 796 netdev_info(vsi->netdev, "NIC Link is Down\n"); 797 return; 798 } 799 800 switch (vsi->port_info->phy.link_info.link_speed) { 801 case ICE_AQ_LINK_SPEED_200GB: 802 speed = "200 G"; 803 break; 804 case ICE_AQ_LINK_SPEED_100GB: 805 speed = "100 G"; 806 break; 807 case ICE_AQ_LINK_SPEED_50GB: 808 speed = "50 G"; 809 break; 810 case ICE_AQ_LINK_SPEED_40GB: 811 speed = "40 G"; 812 break; 813 case ICE_AQ_LINK_SPEED_25GB: 814 speed = "25 G"; 815 break; 816 case ICE_AQ_LINK_SPEED_20GB: 817 speed = "20 G"; 818 break; 819 case ICE_AQ_LINK_SPEED_10GB: 820 speed = "10 G"; 821 break; 822 case ICE_AQ_LINK_SPEED_5GB: 823 speed = "5 G"; 824 break; 825 case ICE_AQ_LINK_SPEED_2500MB: 826 speed = "2.5 G"; 827 break; 828 case ICE_AQ_LINK_SPEED_1000MB: 829 speed = "1 G"; 830 break; 831 case ICE_AQ_LINK_SPEED_100MB: 832 speed = "100 M"; 833 break; 834 default: 835 speed = "Unknown "; 836 break; 837 } 838 839 switch (vsi->port_info->fc.current_mode) { 840 case ICE_FC_FULL: 841 fc = "Rx/Tx"; 842 break; 843 case ICE_FC_TX_PAUSE: 844 fc = "Tx"; 845 break; 846 case ICE_FC_RX_PAUSE: 847 fc = "Rx"; 848 break; 849 case ICE_FC_NONE: 850 fc = "None"; 851 break; 852 default: 853 fc = "Unknown"; 854 break; 855 } 856 857 /* Get FEC mode based on negotiated link info */ 858 switch (vsi->port_info->phy.link_info.fec_info) { 859 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 860 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 861 fec = "RS-FEC"; 862 break; 863 case ICE_AQ_LINK_25G_KR_FEC_EN: 864 fec = "FC-FEC/BASE-R"; 865 break; 866 default: 867 fec = "NONE"; 868 break; 869 } 870 871 /* check if autoneg completed, might be false due to not supported */ 872 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 873 an = "True"; 874 else 875 an = "False"; 876 877 /* Get FEC mode requested based on PHY caps last SW configuration */ 878 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 879 if (!caps) { 880 fec_req = "Unknown"; 881 an_advertised = "Unknown"; 882 goto done; 883 } 884 885 status = ice_aq_get_phy_caps(vsi->port_info, false, 886 ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL); 887 if (status) 888 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 889 890 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 891 892 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 893 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 894 fec_req = "RS-FEC"; 895 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 896 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 897 fec_req = "FC-FEC/BASE-R"; 898 else 899 fec_req = "NONE"; 900 901 kfree(caps); 902 903done: 904 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 905 speed, fec_req, fec, an_advertised, an, fc); 906 ice_print_topo_conflict(vsi); 907} 908 909/** 910 * ice_vsi_link_event - update the VSI's netdev 911 * @vsi: the VSI on which the link event occurred 912 * @link_up: whether or not the VSI needs to be set up or down 913 */ 914static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 915{ 916 if (!vsi) 917 return; 918 919 if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev) 920 return; 921 922 if (vsi->type == ICE_VSI_PF) { 923 if (link_up == netif_carrier_ok(vsi->netdev)) 924 return; 925 926 if (link_up) { 927 netif_carrier_on(vsi->netdev); 928 netif_tx_wake_all_queues(vsi->netdev); 929 } else { 930 netif_carrier_off(vsi->netdev); 931 netif_tx_stop_all_queues(vsi->netdev); 932 } 933 } 934} 935 936/** 937 * ice_set_dflt_mib - send a default config MIB to the FW 938 * @pf: private PF struct 939 * 940 * This function sends a default configuration MIB to the FW. 941 * 942 * If this function errors out at any point, the driver is still able to 943 * function. The main impact is that LFC may not operate as expected. 944 * Therefore an error state in this function should be treated with a DBG 945 * message and continue on with driver rebuild/reenable. 946 */ 947static void ice_set_dflt_mib(struct ice_pf *pf) 948{ 949 struct device *dev = ice_pf_to_dev(pf); 950 u8 mib_type, *buf, *lldpmib = NULL; 951 u16 len, typelen, offset = 0; 952 struct ice_lldp_org_tlv *tlv; 953 struct ice_hw *hw = &pf->hw; 954 u32 ouisubtype; 955 956 mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB; 957 lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL); 958 if (!lldpmib) { 959 dev_dbg(dev, "%s Failed to allocate MIB memory\n", 960 __func__); 961 return; 962 } 963 964 /* Add ETS CFG TLV */ 965 tlv = (struct ice_lldp_org_tlv *)lldpmib; 966 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 967 ICE_IEEE_ETS_TLV_LEN); 968 tlv->typelen = htons(typelen); 969 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 970 ICE_IEEE_SUBTYPE_ETS_CFG); 971 tlv->ouisubtype = htonl(ouisubtype); 972 973 buf = tlv->tlvinfo; 974 buf[0] = 0; 975 976 /* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0. 977 * Octets 5 - 12 are BW values, set octet 5 to 100% BW. 978 * Octets 13 - 20 are TSA values - leave as zeros 979 */ 980 buf[5] = 0x64; 981 len = FIELD_GET(ICE_LLDP_TLV_LEN_M, typelen); 982 offset += len + 2; 983 tlv = (struct ice_lldp_org_tlv *) 984 ((char *)tlv + sizeof(tlv->typelen) + len); 985 986 /* Add ETS REC TLV */ 987 buf = tlv->tlvinfo; 988 tlv->typelen = htons(typelen); 989 990 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 991 ICE_IEEE_SUBTYPE_ETS_REC); 992 tlv->ouisubtype = htonl(ouisubtype); 993 994 /* First octet of buf is reserved 995 * Octets 1 - 4 map UP to TC - all UPs map to zero 996 * Octets 5 - 12 are BW values - set TC 0 to 100%. 997 * Octets 13 - 20 are TSA value - leave as zeros 998 */ 999 buf[5] = 0x64; 1000 offset += len + 2; 1001 tlv = (struct ice_lldp_org_tlv *) 1002 ((char *)tlv + sizeof(tlv->typelen) + len); 1003 1004 /* Add PFC CFG TLV */ 1005 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 1006 ICE_IEEE_PFC_TLV_LEN); 1007 tlv->typelen = htons(typelen); 1008 1009 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 1010 ICE_IEEE_SUBTYPE_PFC_CFG); 1011 tlv->ouisubtype = htonl(ouisubtype); 1012 1013 /* Octet 1 left as all zeros - PFC disabled */ 1014 buf[0] = 0x08; 1015 len = FIELD_GET(ICE_LLDP_TLV_LEN_M, typelen); 1016 offset += len + 2; 1017 1018 if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL)) 1019 dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__); 1020 1021 kfree(lldpmib); 1022} 1023 1024/** 1025 * ice_check_phy_fw_load - check if PHY FW load failed 1026 * @pf: pointer to PF struct 1027 * @link_cfg_err: bitmap from the link info structure 1028 * 1029 * check if external PHY FW load failed and print an error message if it did 1030 */ 1031static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err) 1032{ 1033 if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { 1034 clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1035 return; 1036 } 1037 1038 if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags)) 1039 return; 1040 1041 if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) { 1042 dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n"); 1043 set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1044 } 1045} 1046 1047/** 1048 * ice_check_module_power 1049 * @pf: pointer to PF struct 1050 * @link_cfg_err: bitmap from the link info structure 1051 * 1052 * check module power level returned by a previous call to aq_get_link_info 1053 * and print error messages if module power level is not supported 1054 */ 1055static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err) 1056{ 1057 /* if module power level is supported, clear the flag */ 1058 if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT | 1059 ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) { 1060 clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1061 return; 1062 } 1063 1064 /* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the 1065 * above block didn't clear this bit, there's nothing to do 1066 */ 1067 if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags)) 1068 return; 1069 1070 if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) { 1071 dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n"); 1072 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1073 } else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) { 1074 dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n"); 1075 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1076 } 1077} 1078 1079/** 1080 * ice_check_link_cfg_err - check if link configuration failed 1081 * @pf: pointer to the PF struct 1082 * @link_cfg_err: bitmap from the link info structure 1083 * 1084 * print if any link configuration failure happens due to the value in the 1085 * link_cfg_err parameter in the link info structure 1086 */ 1087static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err) 1088{ 1089 ice_check_module_power(pf, link_cfg_err); 1090 ice_check_phy_fw_load(pf, link_cfg_err); 1091} 1092 1093/** 1094 * ice_link_event - process the link event 1095 * @pf: PF that the link event is associated with 1096 * @pi: port_info for the port that the link event is associated with 1097 * @link_up: true if the physical link is up and false if it is down 1098 * @link_speed: current link speed received from the link event 1099 * 1100 * Returns 0 on success and negative on failure 1101 */ 1102static int 1103ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 1104 u16 link_speed) 1105{ 1106 struct device *dev = ice_pf_to_dev(pf); 1107 struct ice_phy_info *phy_info; 1108 struct ice_vsi *vsi; 1109 u16 old_link_speed; 1110 bool old_link; 1111 int status; 1112 1113 phy_info = &pi->phy; 1114 phy_info->link_info_old = phy_info->link_info; 1115 1116 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 1117 old_link_speed = phy_info->link_info_old.link_speed; 1118 1119 /* update the link info structures and re-enable link events, 1120 * don't bail on failure due to other book keeping needed 1121 */ 1122 status = ice_update_link_info(pi); 1123 if (status) 1124 dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n", 1125 pi->lport, status, 1126 libie_aq_str(pi->hw->adminq.sq_last_status)); 1127 1128 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 1129 1130 /* Check if the link state is up after updating link info, and treat 1131 * this event as an UP event since the link is actually UP now. 1132 */ 1133 if (phy_info->link_info.link_info & ICE_AQ_LINK_UP) 1134 link_up = true; 1135 1136 vsi = ice_get_main_vsi(pf); 1137 if (!vsi || !vsi->port_info) 1138 return -EINVAL; 1139 1140 /* turn off PHY if media was removed */ 1141 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 1142 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 1143 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 1144 ice_set_link(vsi, false); 1145 } 1146 1147 /* if the old link up/down and speed is the same as the new */ 1148 if (link_up == old_link && link_speed == old_link_speed) 1149 return 0; 1150 1151 if (!link_up && old_link) 1152 pf->link_down_events++; 1153 1154 ice_ptp_link_change(pf, link_up); 1155 1156 if (ice_is_dcb_active(pf)) { 1157 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 1158 ice_dcb_rebuild(pf); 1159 } else { 1160 if (link_up) 1161 ice_set_dflt_mib(pf); 1162 } 1163 ice_vsi_link_event(vsi, link_up); 1164 ice_print_link_msg(vsi, link_up); 1165 1166 ice_vc_notify_link_state(pf); 1167 1168 return 0; 1169} 1170 1171/** 1172 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 1173 * @pf: board private structure 1174 */ 1175static void ice_watchdog_subtask(struct ice_pf *pf) 1176{ 1177 int i; 1178 1179 /* if interface is down do nothing */ 1180 if (test_bit(ICE_DOWN, pf->state) || 1181 test_bit(ICE_CFG_BUSY, pf->state)) 1182 return; 1183 1184 /* make sure we don't do these things too often */ 1185 if (time_before(jiffies, 1186 pf->serv_tmr_prev + pf->serv_tmr_period)) 1187 return; 1188 1189 pf->serv_tmr_prev = jiffies; 1190 1191 /* Update the stats for active netdevs so the network stack 1192 * can look at updated numbers whenever it cares to 1193 */ 1194 ice_update_pf_stats(pf); 1195 ice_for_each_vsi(pf, i) 1196 if (pf->vsi[i] && pf->vsi[i]->netdev) 1197 ice_update_vsi_stats(pf->vsi[i]); 1198} 1199 1200/** 1201 * ice_init_link_events - enable/initialize link events 1202 * @pi: pointer to the port_info instance 1203 * 1204 * Returns -EIO on failure, 0 on success 1205 */ 1206static int ice_init_link_events(struct ice_port_info *pi) 1207{ 1208 u16 mask; 1209 1210 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 1211 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL | 1212 ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL)); 1213 1214 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 1215 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 1216 pi->lport); 1217 return -EIO; 1218 } 1219 1220 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 1221 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 1222 pi->lport); 1223 return -EIO; 1224 } 1225 1226 return 0; 1227} 1228 1229/** 1230 * ice_handle_link_event - handle link event via ARQ 1231 * @pf: PF that the link event is associated with 1232 * @event: event structure containing link status info 1233 */ 1234static int 1235ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 1236{ 1237 struct ice_aqc_get_link_status_data *link_data; 1238 struct ice_port_info *port_info; 1239 int status; 1240 1241 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 1242 port_info = pf->hw.port_info; 1243 if (!port_info) 1244 return -EINVAL; 1245 1246 status = ice_link_event(pf, port_info, 1247 !!(link_data->link_info & ICE_AQ_LINK_UP), 1248 le16_to_cpu(link_data->link_speed)); 1249 if (status) 1250 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 1251 status); 1252 1253 return status; 1254} 1255 1256/** 1257 * ice_aq_prep_for_event - Prepare to wait for an AdminQ event from firmware 1258 * @pf: pointer to the PF private structure 1259 * @task: intermediate helper storage and identifier for waiting 1260 * @opcode: the opcode to wait for 1261 * 1262 * Prepares to wait for a specific AdminQ completion event on the ARQ for 1263 * a given PF. Actual wait would be done by a call to ice_aq_wait_for_event(). 1264 * 1265 * Calls are separated to allow caller registering for event before sending 1266 * the command, which mitigates a race between registering and FW responding. 1267 * 1268 * To obtain only the descriptor contents, pass an task->event with null 1269 * msg_buf. If the complete data buffer is desired, allocate the 1270 * task->event.msg_buf with enough space ahead of time. 1271 */ 1272void ice_aq_prep_for_event(struct ice_pf *pf, struct ice_aq_task *task, 1273 u16 opcode) 1274{ 1275 INIT_HLIST_NODE(&task->entry); 1276 task->opcode = opcode; 1277 task->state = ICE_AQ_TASK_WAITING; 1278 1279 spin_lock_bh(&pf->aq_wait_lock); 1280 hlist_add_head(&task->entry, &pf->aq_wait_list); 1281 spin_unlock_bh(&pf->aq_wait_lock); 1282} 1283 1284/** 1285 * ice_aq_wait_for_event - Wait for an AdminQ event from firmware 1286 * @pf: pointer to the PF private structure 1287 * @task: ptr prepared by ice_aq_prep_for_event() 1288 * @timeout: how long to wait, in jiffies 1289 * 1290 * Waits for a specific AdminQ completion event on the ARQ for a given PF. The 1291 * current thread will be put to sleep until the specified event occurs or 1292 * until the given timeout is reached. 1293 * 1294 * Returns: zero on success, or a negative error code on failure. 1295 */ 1296int ice_aq_wait_for_event(struct ice_pf *pf, struct ice_aq_task *task, 1297 unsigned long timeout) 1298{ 1299 enum ice_aq_task_state *state = &task->state; 1300 struct device *dev = ice_pf_to_dev(pf); 1301 unsigned long start = jiffies; 1302 long ret; 1303 int err; 1304 1305 ret = wait_event_interruptible_timeout(pf->aq_wait_queue, 1306 *state != ICE_AQ_TASK_WAITING, 1307 timeout); 1308 switch (*state) { 1309 case ICE_AQ_TASK_NOT_PREPARED: 1310 WARN(1, "call to %s without ice_aq_prep_for_event()", __func__); 1311 err = -EINVAL; 1312 break; 1313 case ICE_AQ_TASK_WAITING: 1314 err = ret < 0 ? ret : -ETIMEDOUT; 1315 break; 1316 case ICE_AQ_TASK_CANCELED: 1317 err = ret < 0 ? ret : -ECANCELED; 1318 break; 1319 case ICE_AQ_TASK_COMPLETE: 1320 err = ret < 0 ? ret : 0; 1321 break; 1322 default: 1323 WARN(1, "Unexpected AdminQ wait task state %u", *state); 1324 err = -EINVAL; 1325 break; 1326 } 1327 1328 dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n", 1329 jiffies_to_msecs(jiffies - start), 1330 jiffies_to_msecs(timeout), 1331 task->opcode); 1332 1333 spin_lock_bh(&pf->aq_wait_lock); 1334 hlist_del(&task->entry); 1335 spin_unlock_bh(&pf->aq_wait_lock); 1336 1337 return err; 1338} 1339 1340/** 1341 * ice_aq_check_events - Check if any thread is waiting for an AdminQ event 1342 * @pf: pointer to the PF private structure 1343 * @opcode: the opcode of the event 1344 * @event: the event to check 1345 * 1346 * Loops over the current list of pending threads waiting for an AdminQ event. 1347 * For each matching task, copy the contents of the event into the task 1348 * structure and wake up the thread. 1349 * 1350 * If multiple threads wait for the same opcode, they will all be woken up. 1351 * 1352 * Note that event->msg_buf will only be duplicated if the event has a buffer 1353 * with enough space already allocated. Otherwise, only the descriptor and 1354 * message length will be copied. 1355 * 1356 * Returns: true if an event was found, false otherwise 1357 */ 1358static void ice_aq_check_events(struct ice_pf *pf, u16 opcode, 1359 struct ice_rq_event_info *event) 1360{ 1361 struct ice_rq_event_info *task_ev; 1362 struct ice_aq_task *task; 1363 bool found = false; 1364 1365 spin_lock_bh(&pf->aq_wait_lock); 1366 hlist_for_each_entry(task, &pf->aq_wait_list, entry) { 1367 if (task->state != ICE_AQ_TASK_WAITING) 1368 continue; 1369 if (task->opcode != opcode) 1370 continue; 1371 1372 task_ev = &task->event; 1373 memcpy(&task_ev->desc, &event->desc, sizeof(event->desc)); 1374 task_ev->msg_len = event->msg_len; 1375 1376 /* Only copy the data buffer if a destination was set */ 1377 if (task_ev->msg_buf && task_ev->buf_len >= event->buf_len) { 1378 memcpy(task_ev->msg_buf, event->msg_buf, 1379 event->buf_len); 1380 task_ev->buf_len = event->buf_len; 1381 } 1382 1383 task->state = ICE_AQ_TASK_COMPLETE; 1384 found = true; 1385 } 1386 spin_unlock_bh(&pf->aq_wait_lock); 1387 1388 if (found) 1389 wake_up(&pf->aq_wait_queue); 1390} 1391 1392/** 1393 * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks 1394 * @pf: the PF private structure 1395 * 1396 * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads. 1397 * This will then cause ice_aq_wait_for_event to exit with -ECANCELED. 1398 */ 1399static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf) 1400{ 1401 struct ice_aq_task *task; 1402 1403 spin_lock_bh(&pf->aq_wait_lock); 1404 hlist_for_each_entry(task, &pf->aq_wait_list, entry) 1405 task->state = ICE_AQ_TASK_CANCELED; 1406 spin_unlock_bh(&pf->aq_wait_lock); 1407 1408 wake_up(&pf->aq_wait_queue); 1409} 1410 1411#define ICE_MBX_OVERFLOW_WATERMARK 64 1412 1413/** 1414 * __ice_clean_ctrlq - helper function to clean controlq rings 1415 * @pf: ptr to struct ice_pf 1416 * @q_type: specific Control queue type 1417 */ 1418static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 1419{ 1420 struct device *dev = ice_pf_to_dev(pf); 1421 struct ice_rq_event_info event; 1422 struct ice_hw *hw = &pf->hw; 1423 struct ice_ctl_q_info *cq; 1424 u16 pending, i = 0; 1425 const char *qtype; 1426 u32 oldval, val; 1427 1428 /* Do not clean control queue if/when PF reset fails */ 1429 if (test_bit(ICE_RESET_FAILED, pf->state)) 1430 return 0; 1431 1432 switch (q_type) { 1433 case ICE_CTL_Q_ADMIN: 1434 cq = &hw->adminq; 1435 qtype = "Admin"; 1436 break; 1437 case ICE_CTL_Q_SB: 1438 cq = &hw->sbq; 1439 qtype = "Sideband"; 1440 break; 1441 case ICE_CTL_Q_MAILBOX: 1442 cq = &hw->mailboxq; 1443 qtype = "Mailbox"; 1444 /* we are going to try to detect a malicious VF, so set the 1445 * state to begin detection 1446 */ 1447 hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT; 1448 break; 1449 default: 1450 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 1451 return 0; 1452 } 1453 1454 /* check for error indications - PF_xx_AxQLEN register layout for 1455 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 1456 */ 1457 val = rd32(hw, cq->rq.len); 1458 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1459 PF_FW_ARQLEN_ARQCRIT_M)) { 1460 oldval = val; 1461 if (val & PF_FW_ARQLEN_ARQVFE_M) 1462 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 1463 qtype); 1464 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 1465 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 1466 qtype); 1467 } 1468 if (val & PF_FW_ARQLEN_ARQCRIT_M) 1469 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 1470 qtype); 1471 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1472 PF_FW_ARQLEN_ARQCRIT_M); 1473 if (oldval != val) 1474 wr32(hw, cq->rq.len, val); 1475 } 1476 1477 val = rd32(hw, cq->sq.len); 1478 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1479 PF_FW_ATQLEN_ATQCRIT_M)) { 1480 oldval = val; 1481 if (val & PF_FW_ATQLEN_ATQVFE_M) 1482 dev_dbg(dev, "%s Send Queue VF Error detected\n", 1483 qtype); 1484 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 1485 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 1486 qtype); 1487 } 1488 if (val & PF_FW_ATQLEN_ATQCRIT_M) 1489 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 1490 qtype); 1491 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1492 PF_FW_ATQLEN_ATQCRIT_M); 1493 if (oldval != val) 1494 wr32(hw, cq->sq.len, val); 1495 } 1496 1497 event.buf_len = cq->rq_buf_size; 1498 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 1499 if (!event.msg_buf) 1500 return 0; 1501 1502 do { 1503 struct ice_mbx_data data = {}; 1504 u16 opcode; 1505 int ret; 1506 1507 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1508 if (ret == -EALREADY) 1509 break; 1510 if (ret) { 1511 dev_err(dev, "%s Receive Queue event error %d\n", qtype, 1512 ret); 1513 break; 1514 } 1515 1516 opcode = le16_to_cpu(event.desc.opcode); 1517 1518 /* Notify any thread that might be waiting for this event */ 1519 ice_aq_check_events(pf, opcode, &event); 1520 1521 switch (opcode) { 1522 case ice_aqc_opc_get_link_status: 1523 if (ice_handle_link_event(pf, &event)) 1524 dev_err(dev, "Could not handle link event\n"); 1525 break; 1526 case ice_aqc_opc_event_lan_overflow: 1527 ice_vf_lan_overflow_event(pf, &event); 1528 break; 1529 case ice_mbx_opc_send_msg_to_pf: 1530 if (ice_is_feature_supported(pf, ICE_F_MBX_LIMIT)) { 1531 ice_vc_process_vf_msg(pf, &event, NULL); 1532 ice_mbx_vf_dec_trig_e830(hw, &event); 1533 } else { 1534 u16 val = hw->mailboxq.num_rq_entries; 1535 1536 data.max_num_msgs_mbx = val; 1537 val = ICE_MBX_OVERFLOW_WATERMARK; 1538 data.async_watermark_val = val; 1539 data.num_msg_proc = i; 1540 data.num_pending_arq = pending; 1541 1542 ice_vc_process_vf_msg(pf, &event, &data); 1543 } 1544 break; 1545 case ice_aqc_opc_fw_logs_event: 1546 libie_get_fwlog_data(&hw->fwlog, event.msg_buf, 1547 le16_to_cpu(event.desc.datalen)); 1548 break; 1549 case ice_aqc_opc_lldp_set_mib_change: 1550 ice_dcb_process_lldp_set_mib_change(pf, &event); 1551 break; 1552 case ice_aqc_opc_get_health_status: 1553 ice_process_health_status_event(pf, &event); 1554 break; 1555 default: 1556 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1557 qtype, opcode); 1558 break; 1559 } 1560 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1561 1562 kfree(event.msg_buf); 1563 1564 return pending && (i == ICE_DFLT_IRQ_WORK); 1565} 1566 1567/** 1568 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1569 * @hw: pointer to hardware info 1570 * @cq: control queue information 1571 * 1572 * returns true if there are pending messages in a queue, false if there aren't 1573 */ 1574static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1575{ 1576 u16 ntu; 1577 1578 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1579 return cq->rq.next_to_clean != ntu; 1580} 1581 1582/** 1583 * ice_clean_adminq_subtask - clean the AdminQ rings 1584 * @pf: board private structure 1585 */ 1586static void ice_clean_adminq_subtask(struct ice_pf *pf) 1587{ 1588 struct ice_hw *hw = &pf->hw; 1589 1590 if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 1591 return; 1592 1593 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1594 return; 1595 1596 clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 1597 1598 /* There might be a situation where new messages arrive to a control 1599 * queue between processing the last message and clearing the 1600 * EVENT_PENDING bit. So before exiting, check queue head again (using 1601 * ice_ctrlq_pending) and process new messages if any. 1602 */ 1603 if (ice_ctrlq_pending(hw, &hw->adminq)) 1604 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1605 1606 ice_flush(hw); 1607} 1608 1609/** 1610 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1611 * @pf: board private structure 1612 */ 1613static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1614{ 1615 struct ice_hw *hw = &pf->hw; 1616 1617 if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1618 return; 1619 1620 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1621 return; 1622 1623 clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1624 1625 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1626 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1627 1628 ice_flush(hw); 1629} 1630 1631/** 1632 * ice_clean_sbq_subtask - clean the Sideband Queue rings 1633 * @pf: board private structure 1634 */ 1635static void ice_clean_sbq_subtask(struct ice_pf *pf) 1636{ 1637 struct ice_hw *hw = &pf->hw; 1638 1639 /* if mac_type is not generic, sideband is not supported 1640 * and there's nothing to do here 1641 */ 1642 if (!ice_is_generic_mac(hw)) { 1643 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1644 return; 1645 } 1646 1647 if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state)) 1648 return; 1649 1650 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB)) 1651 return; 1652 1653 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1654 1655 if (ice_ctrlq_pending(hw, &hw->sbq)) 1656 __ice_clean_ctrlq(pf, ICE_CTL_Q_SB); 1657 1658 ice_flush(hw); 1659} 1660 1661/** 1662 * ice_service_task_schedule - schedule the service task to wake up 1663 * @pf: board private structure 1664 * 1665 * If not already scheduled, this puts the task into the work queue. 1666 */ 1667void ice_service_task_schedule(struct ice_pf *pf) 1668{ 1669 if (!test_bit(ICE_SERVICE_DIS, pf->state) && 1670 !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) && 1671 !test_bit(ICE_NEEDS_RESTART, pf->state)) 1672 queue_work(ice_wq, &pf->serv_task); 1673} 1674 1675/** 1676 * ice_service_task_complete - finish up the service task 1677 * @pf: board private structure 1678 */ 1679static void ice_service_task_complete(struct ice_pf *pf) 1680{ 1681 WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state)); 1682 1683 /* force memory (pf->state) to sync before next service task */ 1684 smp_mb__before_atomic(); 1685 clear_bit(ICE_SERVICE_SCHED, pf->state); 1686} 1687 1688/** 1689 * ice_service_task_stop - stop service task and cancel works 1690 * @pf: board private structure 1691 * 1692 * Return 0 if the ICE_SERVICE_DIS bit was not already set, 1693 * 1 otherwise. 1694 */ 1695static int ice_service_task_stop(struct ice_pf *pf) 1696{ 1697 int ret; 1698 1699 ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state); 1700 1701 if (pf->serv_tmr.function) 1702 timer_delete_sync(&pf->serv_tmr); 1703 if (pf->serv_task.func) 1704 cancel_work_sync(&pf->serv_task); 1705 1706 clear_bit(ICE_SERVICE_SCHED, pf->state); 1707 return ret; 1708} 1709 1710/** 1711 * ice_service_task_restart - restart service task and schedule works 1712 * @pf: board private structure 1713 * 1714 * This function is needed for suspend and resume works (e.g WoL scenario) 1715 */ 1716static void ice_service_task_restart(struct ice_pf *pf) 1717{ 1718 clear_bit(ICE_SERVICE_DIS, pf->state); 1719 ice_service_task_schedule(pf); 1720} 1721 1722/** 1723 * ice_service_timer - timer callback to schedule service task 1724 * @t: pointer to timer_list 1725 */ 1726static void ice_service_timer(struct timer_list *t) 1727{ 1728 struct ice_pf *pf = timer_container_of(pf, t, serv_tmr); 1729 1730 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1731 ice_service_task_schedule(pf); 1732} 1733 1734/** 1735 * ice_mdd_maybe_reset_vf - reset VF after MDD event 1736 * @pf: pointer to the PF structure 1737 * @vf: pointer to the VF structure 1738 * @reset_vf_tx: whether Tx MDD has occurred 1739 * @reset_vf_rx: whether Rx MDD has occurred 1740 * 1741 * Since the queue can get stuck on VF MDD events, the PF can be configured to 1742 * automatically reset the VF by enabling the private ethtool flag 1743 * mdd-auto-reset-vf. 1744 */ 1745static void ice_mdd_maybe_reset_vf(struct ice_pf *pf, struct ice_vf *vf, 1746 bool reset_vf_tx, bool reset_vf_rx) 1747{ 1748 struct device *dev = ice_pf_to_dev(pf); 1749 1750 if (!test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) 1751 return; 1752 1753 /* VF MDD event counters will be cleared by reset, so print the event 1754 * prior to reset. 1755 */ 1756 if (reset_vf_tx) 1757 ice_print_vf_tx_mdd_event(vf); 1758 1759 if (reset_vf_rx) 1760 ice_print_vf_rx_mdd_event(vf); 1761 1762 dev_info(dev, "PF-to-VF reset on PF %d VF %d due to MDD event\n", 1763 pf->hw.pf_id, vf->vf_id); 1764 ice_reset_vf(vf, ICE_VF_RESET_NOTIFY | ICE_VF_RESET_LOCK); 1765} 1766 1767/** 1768 * ice_handle_mdd_event - handle malicious driver detect event 1769 * @pf: pointer to the PF structure 1770 * 1771 * Called from service task. OICR interrupt handler indicates MDD event. 1772 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1773 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1774 * disable the queue, the PF can be configured to reset the VF using ethtool 1775 * private flag mdd-auto-reset-vf. 1776 */ 1777static void ice_handle_mdd_event(struct ice_pf *pf) 1778{ 1779 struct device *dev = ice_pf_to_dev(pf); 1780 struct ice_hw *hw = &pf->hw; 1781 struct ice_vf *vf; 1782 unsigned int bkt; 1783 u32 reg; 1784 1785 if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) { 1786 /* Since the VF MDD event logging is rate limited, check if 1787 * there are pending MDD events. 1788 */ 1789 ice_print_vfs_mdd_events(pf); 1790 return; 1791 } 1792 1793 /* find what triggered an MDD event */ 1794 reg = rd32(hw, GL_MDET_TX_PQM); 1795 if (reg & GL_MDET_TX_PQM_VALID_M) { 1796 u8 pf_num = FIELD_GET(GL_MDET_TX_PQM_PF_NUM_M, reg); 1797 u16 vf_num = FIELD_GET(GL_MDET_TX_PQM_VF_NUM_M, reg); 1798 u8 event = FIELD_GET(GL_MDET_TX_PQM_MAL_TYPE_M, reg); 1799 u16 queue = FIELD_GET(GL_MDET_TX_PQM_QNUM_M, reg); 1800 1801 if (netif_msg_tx_err(pf)) 1802 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1803 event, queue, pf_num, vf_num); 1804 ice_report_mdd_event(pf, ICE_MDD_SRC_TX_PQM, pf_num, vf_num, 1805 event, queue); 1806 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1807 } 1808 1809 reg = rd32(hw, GL_MDET_TX_TCLAN_BY_MAC(hw)); 1810 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1811 u8 pf_num = FIELD_GET(GL_MDET_TX_TCLAN_PF_NUM_M, reg); 1812 u16 vf_num = FIELD_GET(GL_MDET_TX_TCLAN_VF_NUM_M, reg); 1813 u8 event = FIELD_GET(GL_MDET_TX_TCLAN_MAL_TYPE_M, reg); 1814 u16 queue = FIELD_GET(GL_MDET_TX_TCLAN_QNUM_M, reg); 1815 1816 if (netif_msg_tx_err(pf)) 1817 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1818 event, queue, pf_num, vf_num); 1819 ice_report_mdd_event(pf, ICE_MDD_SRC_TX_TCLAN, pf_num, vf_num, 1820 event, queue); 1821 wr32(hw, GL_MDET_TX_TCLAN_BY_MAC(hw), U32_MAX); 1822 } 1823 1824 reg = rd32(hw, GL_MDET_RX); 1825 if (reg & GL_MDET_RX_VALID_M) { 1826 u8 pf_num = FIELD_GET(GL_MDET_RX_PF_NUM_M, reg); 1827 u16 vf_num = FIELD_GET(GL_MDET_RX_VF_NUM_M, reg); 1828 u8 event = FIELD_GET(GL_MDET_RX_MAL_TYPE_M, reg); 1829 u16 queue = FIELD_GET(GL_MDET_RX_QNUM_M, reg); 1830 1831 if (netif_msg_rx_err(pf)) 1832 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1833 event, queue, pf_num, vf_num); 1834 ice_report_mdd_event(pf, ICE_MDD_SRC_RX, pf_num, vf_num, event, 1835 queue); 1836 wr32(hw, GL_MDET_RX, 0xffffffff); 1837 } 1838 1839 /* check to see if this PF caused an MDD event */ 1840 reg = rd32(hw, PF_MDET_TX_PQM); 1841 if (reg & PF_MDET_TX_PQM_VALID_M) { 1842 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1843 if (netif_msg_tx_err(pf)) 1844 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1845 } 1846 1847 reg = rd32(hw, PF_MDET_TX_TCLAN_BY_MAC(hw)); 1848 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1849 wr32(hw, PF_MDET_TX_TCLAN_BY_MAC(hw), 0xffff); 1850 if (netif_msg_tx_err(pf)) 1851 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1852 } 1853 1854 reg = rd32(hw, PF_MDET_RX); 1855 if (reg & PF_MDET_RX_VALID_M) { 1856 wr32(hw, PF_MDET_RX, 0xFFFF); 1857 if (netif_msg_rx_err(pf)) 1858 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1859 } 1860 1861 /* Check to see if one of the VFs caused an MDD event, and then 1862 * increment counters and set print pending 1863 */ 1864 mutex_lock(&pf->vfs.table_lock); 1865 ice_for_each_vf(pf, bkt, vf) { 1866 bool reset_vf_tx = false, reset_vf_rx = false; 1867 1868 reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id)); 1869 if (reg & VP_MDET_TX_PQM_VALID_M) { 1870 wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF); 1871 vf->mdd_tx_events.count++; 1872 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1873 if (netif_msg_tx_err(pf)) 1874 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1875 vf->vf_id); 1876 1877 reset_vf_tx = true; 1878 } 1879 1880 reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id)); 1881 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1882 wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF); 1883 vf->mdd_tx_events.count++; 1884 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1885 if (netif_msg_tx_err(pf)) 1886 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1887 vf->vf_id); 1888 1889 reset_vf_tx = true; 1890 } 1891 1892 reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id)); 1893 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1894 wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF); 1895 vf->mdd_tx_events.count++; 1896 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1897 if (netif_msg_tx_err(pf)) 1898 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1899 vf->vf_id); 1900 1901 reset_vf_tx = true; 1902 } 1903 1904 reg = rd32(hw, VP_MDET_RX(vf->vf_id)); 1905 if (reg & VP_MDET_RX_VALID_M) { 1906 wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF); 1907 vf->mdd_rx_events.count++; 1908 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1909 if (netif_msg_rx_err(pf)) 1910 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1911 vf->vf_id); 1912 1913 reset_vf_rx = true; 1914 } 1915 1916 if (reset_vf_tx || reset_vf_rx) 1917 ice_mdd_maybe_reset_vf(pf, vf, reset_vf_tx, 1918 reset_vf_rx); 1919 } 1920 mutex_unlock(&pf->vfs.table_lock); 1921 1922 ice_print_vfs_mdd_events(pf); 1923} 1924 1925/** 1926 * ice_force_phys_link_state - Force the physical link state 1927 * @vsi: VSI to force the physical link state to up/down 1928 * @link_up: true/false indicates to set the physical link to up/down 1929 * 1930 * Force the physical link state by getting the current PHY capabilities from 1931 * hardware and setting the PHY config based on the determined capabilities. If 1932 * link changes a link event will be triggered because both the Enable Automatic 1933 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1934 * 1935 * Returns 0 on success, negative on failure 1936 */ 1937static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1938{ 1939 struct ice_aqc_get_phy_caps_data *pcaps; 1940 struct ice_aqc_set_phy_cfg_data *cfg; 1941 struct ice_port_info *pi; 1942 struct device *dev; 1943 int retcode; 1944 1945 if (!vsi || !vsi->port_info || !vsi->back) 1946 return -EINVAL; 1947 if (vsi->type != ICE_VSI_PF) 1948 return 0; 1949 1950 dev = ice_pf_to_dev(vsi->back); 1951 1952 pi = vsi->port_info; 1953 1954 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1955 if (!pcaps) 1956 return -ENOMEM; 1957 1958 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 1959 NULL); 1960 if (retcode) { 1961 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1962 vsi->vsi_num, retcode); 1963 retcode = -EIO; 1964 goto out; 1965 } 1966 1967 /* No change in link */ 1968 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1969 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1970 goto out; 1971 1972 /* Use the current user PHY configuration. The current user PHY 1973 * configuration is initialized during probe from PHY capabilities 1974 * software mode, and updated on set PHY configuration. 1975 */ 1976 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1977 if (!cfg) { 1978 retcode = -ENOMEM; 1979 goto out; 1980 } 1981 1982 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1983 if (link_up) 1984 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1985 else 1986 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1987 1988 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1989 if (retcode) { 1990 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1991 vsi->vsi_num, retcode); 1992 retcode = -EIO; 1993 } 1994 1995 kfree(cfg); 1996out: 1997 kfree(pcaps); 1998 return retcode; 1999} 2000 2001/** 2002 * ice_init_nvm_phy_type - Initialize the NVM PHY type 2003 * @pi: port info structure 2004 * 2005 * Initialize nvm_phy_type_[low|high] for link lenient mode support 2006 */ 2007static int ice_init_nvm_phy_type(struct ice_port_info *pi) 2008{ 2009 struct ice_aqc_get_phy_caps_data *pcaps; 2010 struct ice_pf *pf = pi->hw->back; 2011 int err; 2012 2013 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2014 if (!pcaps) 2015 return -ENOMEM; 2016 2017 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA, 2018 pcaps, NULL); 2019 2020 if (err) { 2021 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 2022 goto out; 2023 } 2024 2025 pf->nvm_phy_type_hi = pcaps->phy_type_high; 2026 pf->nvm_phy_type_lo = pcaps->phy_type_low; 2027 2028out: 2029 kfree(pcaps); 2030 return err; 2031} 2032 2033/** 2034 * ice_init_link_dflt_override - Initialize link default override 2035 * @pi: port info structure 2036 * 2037 * Initialize link default override and PHY total port shutdown during probe 2038 */ 2039static void ice_init_link_dflt_override(struct ice_port_info *pi) 2040{ 2041 struct ice_link_default_override_tlv *ldo; 2042 struct ice_pf *pf = pi->hw->back; 2043 2044 ldo = &pf->link_dflt_override; 2045 if (ice_get_link_default_override(ldo, pi)) 2046 return; 2047 2048 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 2049 return; 2050 2051 /* Enable Total Port Shutdown (override/replace link-down-on-close 2052 * ethtool private flag) for ports with Port Disable bit set. 2053 */ 2054 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 2055 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 2056} 2057 2058/** 2059 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 2060 * @pi: port info structure 2061 * 2062 * If default override is enabled, initialize the user PHY cfg speed and FEC 2063 * settings using the default override mask from the NVM. 2064 * 2065 * The PHY should only be configured with the default override settings the 2066 * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state 2067 * is used to indicate that the user PHY cfg default override is initialized 2068 * and the PHY has not been configured with the default override settings. The 2069 * state is set here, and cleared in ice_configure_phy the first time the PHY is 2070 * configured. 2071 * 2072 * This function should be called only if the FW doesn't support default 2073 * configuration mode, as reported by ice_fw_supports_report_dflt_cfg. 2074 */ 2075static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 2076{ 2077 struct ice_link_default_override_tlv *ldo; 2078 struct ice_aqc_set_phy_cfg_data *cfg; 2079 struct ice_phy_info *phy = &pi->phy; 2080 struct ice_pf *pf = pi->hw->back; 2081 2082 ldo = &pf->link_dflt_override; 2083 2084 /* If link default override is enabled, use to mask NVM PHY capabilities 2085 * for speed and FEC default configuration. 2086 */ 2087 cfg = &phy->curr_user_phy_cfg; 2088 2089 if (ldo->phy_type_low || ldo->phy_type_high) { 2090 cfg->phy_type_low = pf->nvm_phy_type_lo & 2091 cpu_to_le64(ldo->phy_type_low); 2092 cfg->phy_type_high = pf->nvm_phy_type_hi & 2093 cpu_to_le64(ldo->phy_type_high); 2094 } 2095 cfg->link_fec_opt = ldo->fec_options; 2096 phy->curr_user_fec_req = ICE_FEC_AUTO; 2097 2098 set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 2099} 2100 2101/** 2102 * ice_init_phy_user_cfg - Initialize the PHY user configuration 2103 * @pi: port info structure 2104 * 2105 * Initialize the current user PHY configuration, speed, FEC, and FC requested 2106 * mode to default. The PHY defaults are from get PHY capabilities topology 2107 * with media so call when media is first available. An error is returned if 2108 * called when media is not available. The PHY initialization completed state is 2109 * set here. 2110 * 2111 * These configurations are used when setting PHY 2112 * configuration. The user PHY configuration is updated on set PHY 2113 * configuration. Returns 0 on success, negative on failure 2114 */ 2115static int ice_init_phy_user_cfg(struct ice_port_info *pi) 2116{ 2117 struct ice_aqc_get_phy_caps_data *pcaps; 2118 struct ice_phy_info *phy = &pi->phy; 2119 struct ice_pf *pf = pi->hw->back; 2120 int err; 2121 2122 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2123 return -EIO; 2124 2125 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2126 if (!pcaps) 2127 return -ENOMEM; 2128 2129 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2130 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2131 pcaps, NULL); 2132 else 2133 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2134 pcaps, NULL); 2135 if (err) { 2136 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 2137 goto err_out; 2138 } 2139 2140 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 2141 2142 /* check if lenient mode is supported and enabled */ 2143 if (ice_fw_supports_link_override(pi->hw) && 2144 !(pcaps->module_compliance_enforcement & 2145 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 2146 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 2147 2148 /* if the FW supports default PHY configuration mode, then the driver 2149 * does not have to apply link override settings. If not, 2150 * initialize user PHY configuration with link override values 2151 */ 2152 if (!ice_fw_supports_report_dflt_cfg(pi->hw) && 2153 (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) { 2154 ice_init_phy_cfg_dflt_override(pi); 2155 goto out; 2156 } 2157 } 2158 2159 /* if link default override is not enabled, set user flow control and 2160 * FEC settings based on what get_phy_caps returned 2161 */ 2162 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 2163 pcaps->link_fec_options); 2164 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 2165 2166out: 2167 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 2168 set_bit(ICE_PHY_INIT_COMPLETE, pf->state); 2169err_out: 2170 kfree(pcaps); 2171 return err; 2172} 2173 2174/** 2175 * ice_configure_phy - configure PHY 2176 * @vsi: VSI of PHY 2177 * 2178 * Set the PHY configuration. If the current PHY configuration is the same as 2179 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 2180 * configure the based get PHY capabilities for topology with media. 2181 */ 2182static int ice_configure_phy(struct ice_vsi *vsi) 2183{ 2184 struct device *dev = ice_pf_to_dev(vsi->back); 2185 struct ice_port_info *pi = vsi->port_info; 2186 struct ice_aqc_get_phy_caps_data *pcaps; 2187 struct ice_aqc_set_phy_cfg_data *cfg; 2188 struct ice_phy_info *phy = &pi->phy; 2189 struct ice_pf *pf = vsi->back; 2190 int err; 2191 2192 /* Ensure we have media as we cannot configure a medialess port */ 2193 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2194 return -ENOMEDIUM; 2195 2196 ice_print_topo_conflict(vsi); 2197 2198 if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) && 2199 phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 2200 return -EPERM; 2201 2202 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) 2203 return ice_force_phys_link_state(vsi, true); 2204 2205 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2206 if (!pcaps) 2207 return -ENOMEM; 2208 2209 /* Get current PHY config */ 2210 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 2211 NULL); 2212 if (err) { 2213 dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n", 2214 vsi->vsi_num, err); 2215 goto done; 2216 } 2217 2218 /* If PHY enable link is configured and configuration has not changed, 2219 * there's nothing to do 2220 */ 2221 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 2222 ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg)) 2223 goto done; 2224 2225 /* Use PHY topology as baseline for configuration */ 2226 memset(pcaps, 0, sizeof(*pcaps)); 2227 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2228 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2229 pcaps, NULL); 2230 else 2231 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2232 pcaps, NULL); 2233 if (err) { 2234 dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n", 2235 vsi->vsi_num, err); 2236 goto done; 2237 } 2238 2239 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 2240 if (!cfg) { 2241 err = -ENOMEM; 2242 goto done; 2243 } 2244 2245 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 2246 2247 /* Speed - If default override pending, use curr_user_phy_cfg set in 2248 * ice_init_phy_user_cfg_ldo. 2249 */ 2250 if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, 2251 vsi->back->state)) { 2252 cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low; 2253 cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high; 2254 } else { 2255 u64 phy_low = 0, phy_high = 0; 2256 2257 ice_update_phy_type(&phy_low, &phy_high, 2258 pi->phy.curr_user_speed_req); 2259 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 2260 cfg->phy_type_high = pcaps->phy_type_high & 2261 cpu_to_le64(phy_high); 2262 } 2263 2264 /* Can't provide what was requested; use PHY capabilities */ 2265 if (!cfg->phy_type_low && !cfg->phy_type_high) { 2266 cfg->phy_type_low = pcaps->phy_type_low; 2267 cfg->phy_type_high = pcaps->phy_type_high; 2268 } 2269 2270 /* FEC */ 2271 ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req); 2272 2273 /* Can't provide what was requested; use PHY capabilities */ 2274 if (cfg->link_fec_opt != 2275 (cfg->link_fec_opt & pcaps->link_fec_options)) { 2276 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 2277 cfg->link_fec_opt = pcaps->link_fec_options; 2278 } 2279 2280 /* Flow Control - always supported; no need to check against 2281 * capabilities 2282 */ 2283 ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req); 2284 2285 /* Enable link and link update */ 2286 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 2287 2288 err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL); 2289 if (err) 2290 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 2291 vsi->vsi_num, err); 2292 2293 kfree(cfg); 2294done: 2295 kfree(pcaps); 2296 return err; 2297} 2298 2299/** 2300 * ice_check_media_subtask - Check for media 2301 * @pf: pointer to PF struct 2302 * 2303 * If media is available, then initialize PHY user configuration if it is not 2304 * been, and configure the PHY if the interface is up. 2305 */ 2306static void ice_check_media_subtask(struct ice_pf *pf) 2307{ 2308 struct ice_port_info *pi; 2309 struct ice_vsi *vsi; 2310 int err; 2311 2312 /* No need to check for media if it's already present */ 2313 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 2314 return; 2315 2316 vsi = ice_get_main_vsi(pf); 2317 if (!vsi) 2318 return; 2319 2320 /* Refresh link info and check if media is present */ 2321 pi = vsi->port_info; 2322 err = ice_update_link_info(pi); 2323 if (err) 2324 return; 2325 2326 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 2327 2328 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2329 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) 2330 ice_init_phy_user_cfg(pi); 2331 2332 /* PHY settings are reset on media insertion, reconfigure 2333 * PHY to preserve settings. 2334 */ 2335 if (test_bit(ICE_VSI_DOWN, vsi->state) && 2336 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 2337 return; 2338 2339 err = ice_configure_phy(vsi); 2340 if (!err) 2341 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 2342 2343 /* A Link Status Event will be generated; the event handler 2344 * will complete bringing the interface up 2345 */ 2346 } 2347} 2348 2349static void ice_service_task_recovery_mode(struct work_struct *work) 2350{ 2351 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 2352 2353 set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 2354 ice_clean_adminq_subtask(pf); 2355 2356 ice_service_task_complete(pf); 2357 2358 mod_timer(&pf->serv_tmr, jiffies + msecs_to_jiffies(100)); 2359} 2360 2361/** 2362 * ice_service_task - manage and run subtasks 2363 * @work: pointer to work_struct contained by the PF struct 2364 */ 2365static void ice_service_task(struct work_struct *work) 2366{ 2367 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 2368 unsigned long start_time = jiffies; 2369 2370 if (pf->health_reporters.tx_hang_buf.tx_ring) { 2371 ice_report_tx_hang(pf); 2372 pf->health_reporters.tx_hang_buf.tx_ring = NULL; 2373 } 2374 2375 ice_reset_subtask(pf); 2376 2377 /* bail if a reset/recovery cycle is pending or rebuild failed */ 2378 if (ice_is_reset_in_progress(pf->state) || 2379 test_bit(ICE_SUSPENDED, pf->state) || 2380 test_bit(ICE_NEEDS_RESTART, pf->state)) { 2381 ice_service_task_complete(pf); 2382 return; 2383 } 2384 2385 if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) { 2386 struct iidc_rdma_event *event; 2387 2388 event = kzalloc(sizeof(*event), GFP_KERNEL); 2389 if (event) { 2390 set_bit(IIDC_RDMA_EVENT_CRIT_ERR, event->type); 2391 /* report the entire OICR value to AUX driver */ 2392 swap(event->reg, pf->oicr_err_reg); 2393 ice_send_event_to_aux(pf, event); 2394 kfree(event); 2395 } 2396 } 2397 2398 /* unplug aux dev per request, if an unplug request came in 2399 * while processing a plug request, this will handle it 2400 */ 2401 if (test_and_clear_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags)) 2402 ice_unplug_aux_dev(pf); 2403 2404 /* Plug aux device per request */ 2405 if (test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags)) 2406 ice_plug_aux_dev(pf); 2407 2408 if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) { 2409 struct iidc_rdma_event *event; 2410 2411 event = kzalloc(sizeof(*event), GFP_KERNEL); 2412 if (event) { 2413 set_bit(IIDC_RDMA_EVENT_AFTER_MTU_CHANGE, event->type); 2414 ice_send_event_to_aux(pf, event); 2415 kfree(event); 2416 } 2417 } 2418 2419 ice_clean_adminq_subtask(pf); 2420 ice_check_media_subtask(pf); 2421 ice_check_for_hang_subtask(pf); 2422 ice_sync_fltr_subtask(pf); 2423 ice_handle_mdd_event(pf); 2424 ice_watchdog_subtask(pf); 2425 2426 if (ice_is_safe_mode(pf)) { 2427 ice_service_task_complete(pf); 2428 return; 2429 } 2430 2431 ice_process_vflr_event(pf); 2432 ice_clean_mailboxq_subtask(pf); 2433 ice_clean_sbq_subtask(pf); 2434 ice_sync_arfs_fltrs(pf); 2435 ice_flush_fdir_ctx(pf); 2436 2437 /* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */ 2438 ice_service_task_complete(pf); 2439 2440 /* If the tasks have taken longer than one service timer period 2441 * or there is more work to be done, reset the service timer to 2442 * schedule the service task now. 2443 */ 2444 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 2445 test_bit(ICE_MDD_EVENT_PENDING, pf->state) || 2446 test_bit(ICE_VFLR_EVENT_PENDING, pf->state) || 2447 test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 2448 test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) || 2449 test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) || 2450 test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 2451 mod_timer(&pf->serv_tmr, jiffies); 2452} 2453 2454/** 2455 * ice_set_ctrlq_len - helper function to set controlq length 2456 * @hw: pointer to the HW instance 2457 */ 2458static void ice_set_ctrlq_len(struct ice_hw *hw) 2459{ 2460 hw->adminq.num_rq_entries = ICE_AQ_LEN; 2461 hw->adminq.num_sq_entries = ICE_AQ_LEN; 2462 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 2463 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 2464 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 2465 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 2466 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2467 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2468 hw->sbq.num_rq_entries = ICE_SBQ_LEN; 2469 hw->sbq.num_sq_entries = ICE_SBQ_LEN; 2470 hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2471 hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2472} 2473 2474/** 2475 * ice_schedule_reset - schedule a reset 2476 * @pf: board private structure 2477 * @reset: reset being requested 2478 */ 2479int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 2480{ 2481 struct device *dev = ice_pf_to_dev(pf); 2482 2483 /* bail out if earlier reset has failed */ 2484 if (test_bit(ICE_RESET_FAILED, pf->state)) { 2485 dev_dbg(dev, "earlier reset has failed\n"); 2486 return -EIO; 2487 } 2488 /* bail if reset/recovery already in progress */ 2489 if (ice_is_reset_in_progress(pf->state)) { 2490 dev_dbg(dev, "Reset already in progress\n"); 2491 return -EBUSY; 2492 } 2493 2494 switch (reset) { 2495 case ICE_RESET_PFR: 2496 set_bit(ICE_PFR_REQ, pf->state); 2497 break; 2498 case ICE_RESET_CORER: 2499 set_bit(ICE_CORER_REQ, pf->state); 2500 break; 2501 case ICE_RESET_GLOBR: 2502 set_bit(ICE_GLOBR_REQ, pf->state); 2503 break; 2504 default: 2505 return -EINVAL; 2506 } 2507 2508 ice_service_task_schedule(pf); 2509 return 0; 2510} 2511 2512/** 2513 * ice_vsi_ena_irq - Enable IRQ for the given VSI 2514 * @vsi: the VSI being configured 2515 */ 2516static int ice_vsi_ena_irq(struct ice_vsi *vsi) 2517{ 2518 struct ice_hw *hw = &vsi->back->hw; 2519 int i; 2520 2521 ice_for_each_q_vector(vsi, i) 2522 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 2523 2524 ice_flush(hw); 2525 return 0; 2526} 2527 2528/** 2529 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 2530 * @vsi: the VSI being configured 2531 * @basename: name for the vector 2532 */ 2533static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 2534{ 2535 int q_vectors = vsi->num_q_vectors; 2536 struct ice_pf *pf = vsi->back; 2537 struct device *dev; 2538 int rx_int_idx = 0; 2539 int tx_int_idx = 0; 2540 int vector, err; 2541 int irq_num; 2542 2543 dev = ice_pf_to_dev(pf); 2544 for (vector = 0; vector < q_vectors; vector++) { 2545 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 2546 2547 irq_num = q_vector->irq.virq; 2548 2549 if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) { 2550 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2551 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 2552 tx_int_idx++; 2553 } else if (q_vector->rx.rx_ring) { 2554 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2555 "%s-%s-%d", basename, "rx", rx_int_idx++); 2556 } else if (q_vector->tx.tx_ring) { 2557 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2558 "%s-%s-%d", basename, "tx", tx_int_idx++); 2559 } else { 2560 /* skip this unused q_vector */ 2561 continue; 2562 } 2563 if (vsi->type == ICE_VSI_CTRL && vsi->vf) 2564 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2565 IRQF_SHARED, q_vector->name, 2566 q_vector); 2567 else 2568 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2569 0, q_vector->name, q_vector); 2570 if (err) { 2571 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 2572 err); 2573 goto free_q_irqs; 2574 } 2575 } 2576 2577 err = ice_set_cpu_rx_rmap(vsi); 2578 if (err) { 2579 netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n", 2580 vsi->vsi_num, ERR_PTR(err)); 2581 goto free_q_irqs; 2582 } 2583 2584 vsi->irqs_ready = true; 2585 return 0; 2586 2587free_q_irqs: 2588 while (vector--) { 2589 irq_num = vsi->q_vectors[vector]->irq.virq; 2590 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 2591 } 2592 return err; 2593} 2594 2595/** 2596 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 2597 * @vsi: VSI to setup Tx rings used by XDP 2598 * 2599 * Return 0 on success and negative value on error 2600 */ 2601static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 2602{ 2603 struct device *dev = ice_pf_to_dev(vsi->back); 2604 struct ice_tx_desc *tx_desc; 2605 int i, j; 2606 2607 ice_for_each_xdp_txq(vsi, i) { 2608 u16 xdp_q_idx = vsi->alloc_txq + i; 2609 struct ice_ring_stats *ring_stats; 2610 struct ice_tx_ring *xdp_ring; 2611 2612 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2613 if (!xdp_ring) 2614 goto free_xdp_rings; 2615 2616 ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL); 2617 if (!ring_stats) { 2618 ice_free_tx_ring(xdp_ring); 2619 goto free_xdp_rings; 2620 } 2621 2622 xdp_ring->ring_stats = ring_stats; 2623 xdp_ring->q_index = xdp_q_idx; 2624 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2625 xdp_ring->vsi = vsi; 2626 xdp_ring->netdev = NULL; 2627 xdp_ring->dev = dev; 2628 xdp_ring->count = vsi->num_tx_desc; 2629 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2630 if (ice_setup_tx_ring(xdp_ring)) 2631 goto free_xdp_rings; 2632 ice_set_ring_xdp(xdp_ring); 2633 spin_lock_init(&xdp_ring->tx_lock); 2634 for (j = 0; j < xdp_ring->count; j++) { 2635 tx_desc = ICE_TX_DESC(xdp_ring, j); 2636 tx_desc->cmd_type_offset_bsz = 0; 2637 } 2638 } 2639 2640 return 0; 2641 2642free_xdp_rings: 2643 for (; i >= 0; i--) { 2644 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) { 2645 kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu); 2646 vsi->xdp_rings[i]->ring_stats = NULL; 2647 ice_free_tx_ring(vsi->xdp_rings[i]); 2648 } 2649 } 2650 return -ENOMEM; 2651} 2652 2653/** 2654 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2655 * @vsi: VSI to set the bpf prog on 2656 * @prog: the bpf prog pointer 2657 */ 2658static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2659{ 2660 struct bpf_prog *old_prog; 2661 int i; 2662 2663 old_prog = xchg(&vsi->xdp_prog, prog); 2664 ice_for_each_rxq(vsi, i) 2665 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2666 2667 if (old_prog) 2668 bpf_prog_put(old_prog); 2669} 2670 2671static struct ice_tx_ring *ice_xdp_ring_from_qid(struct ice_vsi *vsi, int qid) 2672{ 2673 struct ice_q_vector *q_vector; 2674 struct ice_tx_ring *ring; 2675 2676 if (static_key_enabled(&ice_xdp_locking_key)) 2677 return vsi->xdp_rings[qid % vsi->num_xdp_txq]; 2678 2679 q_vector = vsi->rx_rings[qid]->q_vector; 2680 ice_for_each_tx_ring(ring, q_vector->tx) 2681 if (ice_ring_is_xdp(ring)) 2682 return ring; 2683 2684 return NULL; 2685} 2686 2687/** 2688 * ice_map_xdp_rings - Map XDP rings to interrupt vectors 2689 * @vsi: the VSI with XDP rings being configured 2690 * 2691 * Map XDP rings to interrupt vectors and perform the configuration steps 2692 * dependent on the mapping. 2693 */ 2694void ice_map_xdp_rings(struct ice_vsi *vsi) 2695{ 2696 int xdp_rings_rem = vsi->num_xdp_txq; 2697 int v_idx, q_idx; 2698 2699 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2700 ice_for_each_q_vector(vsi, v_idx) { 2701 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2702 int xdp_rings_per_v, q_id, q_base; 2703 2704 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2705 vsi->num_q_vectors - v_idx); 2706 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2707 2708 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2709 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id]; 2710 2711 xdp_ring->q_vector = q_vector; 2712 xdp_ring->next = q_vector->tx.tx_ring; 2713 q_vector->tx.tx_ring = xdp_ring; 2714 } 2715 xdp_rings_rem -= xdp_rings_per_v; 2716 } 2717 2718 ice_for_each_rxq(vsi, q_idx) { 2719 vsi->rx_rings[q_idx]->xdp_ring = ice_xdp_ring_from_qid(vsi, 2720 q_idx); 2721 ice_tx_xsk_pool(vsi, q_idx); 2722 } 2723} 2724 2725/** 2726 * ice_unmap_xdp_rings - Unmap XDP rings from interrupt vectors 2727 * @vsi: the VSI with XDP rings being unmapped 2728 */ 2729static void ice_unmap_xdp_rings(struct ice_vsi *vsi) 2730{ 2731 int v_idx; 2732 2733 ice_for_each_q_vector(vsi, v_idx) { 2734 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2735 struct ice_tx_ring *ring; 2736 2737 ice_for_each_tx_ring(ring, q_vector->tx) 2738 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2739 break; 2740 2741 /* restore the value of last node prior to XDP setup */ 2742 q_vector->tx.tx_ring = ring; 2743 } 2744} 2745 2746/** 2747 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2748 * @vsi: VSI to bring up Tx rings used by XDP 2749 * @prog: bpf program that will be assigned to VSI 2750 * @cfg_type: create from scratch or restore the existing configuration 2751 * 2752 * Return 0 on success and negative value on error 2753 */ 2754int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog, 2755 enum ice_xdp_cfg cfg_type) 2756{ 2757 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2758 struct ice_pf *pf = vsi->back; 2759 struct ice_qs_cfg xdp_qs_cfg = { 2760 .qs_mutex = &pf->avail_q_mutex, 2761 .pf_map = pf->avail_txqs, 2762 .pf_map_size = pf->max_pf_txqs, 2763 .q_count = vsi->num_xdp_txq, 2764 .scatter_count = ICE_MAX_SCATTER_TXQS, 2765 .vsi_map = vsi->txq_map, 2766 .vsi_map_offset = vsi->alloc_txq, 2767 .mapping_mode = ICE_VSI_MAP_CONTIG 2768 }; 2769 struct device *dev; 2770 int status, i; 2771 2772 dev = ice_pf_to_dev(pf); 2773 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2774 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2775 if (!vsi->xdp_rings) 2776 return -ENOMEM; 2777 2778 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2779 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2780 goto err_map_xdp; 2781 2782 if (static_key_enabled(&ice_xdp_locking_key)) 2783 netdev_warn(vsi->netdev, 2784 "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n"); 2785 2786 if (ice_xdp_alloc_setup_rings(vsi)) 2787 goto clear_xdp_rings; 2788 2789 /* omit the scheduler update if in reset path; XDP queues will be 2790 * taken into account at the end of ice_vsi_rebuild, where 2791 * ice_cfg_vsi_lan is being called 2792 */ 2793 if (cfg_type == ICE_XDP_CFG_PART) 2794 return 0; 2795 2796 ice_map_xdp_rings(vsi); 2797 2798 /* tell the Tx scheduler that right now we have 2799 * additional queues 2800 */ 2801 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2802 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2803 2804 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2805 max_txqs); 2806 if (status) { 2807 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n", 2808 status); 2809 goto unmap_xdp_rings; 2810 } 2811 2812 /* assign the prog only when it's not already present on VSI; 2813 * this flow is a subject of both ethtool -L and ndo_bpf flows; 2814 * VSI rebuild that happens under ethtool -L can expose us to 2815 * the bpf_prog refcount issues as we would be swapping same 2816 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put 2817 * on it as it would be treated as an 'old_prog'; for ndo_bpf 2818 * this is not harmful as dev_xdp_install bumps the refcount 2819 * before calling the op exposed by the driver; 2820 */ 2821 if (!ice_is_xdp_ena_vsi(vsi)) 2822 ice_vsi_assign_bpf_prog(vsi, prog); 2823 2824 return 0; 2825unmap_xdp_rings: 2826 ice_unmap_xdp_rings(vsi); 2827clear_xdp_rings: 2828 ice_for_each_xdp_txq(vsi, i) 2829 if (vsi->xdp_rings[i]) { 2830 kfree_rcu(vsi->xdp_rings[i], rcu); 2831 vsi->xdp_rings[i] = NULL; 2832 } 2833 2834err_map_xdp: 2835 mutex_lock(&pf->avail_q_mutex); 2836 ice_for_each_xdp_txq(vsi, i) { 2837 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2838 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2839 } 2840 mutex_unlock(&pf->avail_q_mutex); 2841 2842 devm_kfree(dev, vsi->xdp_rings); 2843 vsi->xdp_rings = NULL; 2844 2845 return -ENOMEM; 2846} 2847 2848/** 2849 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2850 * @vsi: VSI to remove XDP rings 2851 * @cfg_type: disable XDP permanently or allow it to be restored later 2852 * 2853 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2854 * resources 2855 */ 2856int ice_destroy_xdp_rings(struct ice_vsi *vsi, enum ice_xdp_cfg cfg_type) 2857{ 2858 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2859 struct ice_pf *pf = vsi->back; 2860 int i; 2861 2862 /* q_vectors are freed in reset path so there's no point in detaching 2863 * rings 2864 */ 2865 if (cfg_type == ICE_XDP_CFG_PART) 2866 goto free_qmap; 2867 2868 ice_unmap_xdp_rings(vsi); 2869 2870free_qmap: 2871 mutex_lock(&pf->avail_q_mutex); 2872 ice_for_each_xdp_txq(vsi, i) { 2873 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2874 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2875 } 2876 mutex_unlock(&pf->avail_q_mutex); 2877 2878 ice_for_each_xdp_txq(vsi, i) 2879 if (vsi->xdp_rings[i]) { 2880 if (vsi->xdp_rings[i]->desc) { 2881 synchronize_rcu(); 2882 ice_free_tx_ring(vsi->xdp_rings[i]); 2883 } 2884 kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu); 2885 vsi->xdp_rings[i]->ring_stats = NULL; 2886 kfree_rcu(vsi->xdp_rings[i], rcu); 2887 vsi->xdp_rings[i] = NULL; 2888 } 2889 2890 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2891 vsi->xdp_rings = NULL; 2892 2893 if (static_key_enabled(&ice_xdp_locking_key)) 2894 static_branch_dec(&ice_xdp_locking_key); 2895 2896 if (cfg_type == ICE_XDP_CFG_PART) 2897 return 0; 2898 2899 ice_vsi_assign_bpf_prog(vsi, NULL); 2900 2901 /* notify Tx scheduler that we destroyed XDP queues and bring 2902 * back the old number of child nodes 2903 */ 2904 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2905 max_txqs[i] = vsi->num_txq; 2906 2907 /* change number of XDP Tx queues to 0 */ 2908 vsi->num_xdp_txq = 0; 2909 2910 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2911 max_txqs); 2912} 2913 2914/** 2915 * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI 2916 * @vsi: VSI to schedule napi on 2917 */ 2918static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi) 2919{ 2920 int i; 2921 2922 ice_for_each_rxq(vsi, i) { 2923 struct ice_rx_ring *rx_ring = vsi->rx_rings[i]; 2924 2925 if (READ_ONCE(rx_ring->xsk_pool)) 2926 napi_schedule(&rx_ring->q_vector->napi); 2927 } 2928} 2929 2930/** 2931 * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have 2932 * @vsi: VSI to determine the count of XDP Tx qs 2933 * 2934 * returns 0 if Tx qs count is higher than at least half of CPU count, 2935 * -ENOMEM otherwise 2936 */ 2937int ice_vsi_determine_xdp_res(struct ice_vsi *vsi) 2938{ 2939 u16 avail = ice_get_avail_txq_count(vsi->back); 2940 u16 cpus = num_possible_cpus(); 2941 2942 if (avail < cpus / 2) 2943 return -ENOMEM; 2944 2945 if (vsi->type == ICE_VSI_SF) 2946 avail = vsi->alloc_txq; 2947 2948 vsi->num_xdp_txq = min_t(u16, avail, cpus); 2949 2950 if (vsi->num_xdp_txq < cpus) 2951 static_branch_inc(&ice_xdp_locking_key); 2952 2953 return 0; 2954} 2955 2956/** 2957 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2958 * @vsi: Pointer to VSI structure 2959 */ 2960static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 2961{ 2962 return ICE_RXBUF_3072; 2963} 2964 2965/** 2966 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2967 * @vsi: VSI to setup XDP for 2968 * @prog: XDP program 2969 * @extack: netlink extended ack 2970 */ 2971static int 2972ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2973 struct netlink_ext_ack *extack) 2974{ 2975 unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2976 int ret = 0, xdp_ring_err = 0; 2977 bool if_running; 2978 2979 if (prog && !prog->aux->xdp_has_frags) { 2980 if (frame_size > ice_max_xdp_frame_size(vsi)) { 2981 NL_SET_ERR_MSG_MOD(extack, 2982 "MTU is too large for linear frames and XDP prog does not support frags"); 2983 return -EOPNOTSUPP; 2984 } 2985 } 2986 2987 /* hot swap progs and avoid toggling link */ 2988 if (ice_is_xdp_ena_vsi(vsi) == !!prog || 2989 test_bit(ICE_VSI_REBUILD_PENDING, vsi->state)) { 2990 ice_vsi_assign_bpf_prog(vsi, prog); 2991 return 0; 2992 } 2993 2994 if_running = netif_running(vsi->netdev) && 2995 !test_and_set_bit(ICE_VSI_DOWN, vsi->state); 2996 2997 /* need to stop netdev while setting up the program for Rx rings */ 2998 if (if_running) { 2999 ret = ice_down(vsi); 3000 if (ret) { 3001 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 3002 return ret; 3003 } 3004 } 3005 3006 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 3007 xdp_ring_err = ice_vsi_determine_xdp_res(vsi); 3008 if (xdp_ring_err) { 3009 NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP"); 3010 goto resume_if; 3011 } else { 3012 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog, 3013 ICE_XDP_CFG_FULL); 3014 if (xdp_ring_err) { 3015 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 3016 goto resume_if; 3017 } 3018 } 3019 xdp_features_set_redirect_target(vsi->netdev, true); 3020 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 3021 xdp_features_clear_redirect_target(vsi->netdev); 3022 xdp_ring_err = ice_destroy_xdp_rings(vsi, ICE_XDP_CFG_FULL); 3023 if (xdp_ring_err) 3024 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 3025 } 3026 3027resume_if: 3028 if (if_running) 3029 ret = ice_up(vsi); 3030 3031 if (!ret && prog) 3032 ice_vsi_rx_napi_schedule(vsi); 3033 3034 return (ret || xdp_ring_err) ? -ENOMEM : 0; 3035} 3036 3037/** 3038 * ice_xdp_safe_mode - XDP handler for safe mode 3039 * @dev: netdevice 3040 * @xdp: XDP command 3041 */ 3042static int ice_xdp_safe_mode(struct net_device __always_unused *dev, 3043 struct netdev_bpf *xdp) 3044{ 3045 NL_SET_ERR_MSG_MOD(xdp->extack, 3046 "Please provide working DDP firmware package in order to use XDP\n" 3047 "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst"); 3048 return -EOPNOTSUPP; 3049} 3050 3051/** 3052 * ice_xdp - implements XDP handler 3053 * @dev: netdevice 3054 * @xdp: XDP command 3055 */ 3056int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 3057{ 3058 struct ice_netdev_priv *np = netdev_priv(dev); 3059 struct ice_vsi *vsi = np->vsi; 3060 int ret; 3061 3062 if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_SF) { 3063 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF or SF VSI"); 3064 return -EINVAL; 3065 } 3066 3067 mutex_lock(&vsi->xdp_state_lock); 3068 3069 switch (xdp->command) { 3070 case XDP_SETUP_PROG: 3071 ret = ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 3072 break; 3073 case XDP_SETUP_XSK_POOL: 3074 ret = ice_xsk_pool_setup(vsi, xdp->xsk.pool, xdp->xsk.queue_id); 3075 break; 3076 default: 3077 ret = -EINVAL; 3078 } 3079 3080 mutex_unlock(&vsi->xdp_state_lock); 3081 return ret; 3082} 3083 3084/** 3085 * ice_ena_misc_vector - enable the non-queue interrupts 3086 * @pf: board private structure 3087 */ 3088static void ice_ena_misc_vector(struct ice_pf *pf) 3089{ 3090 struct ice_hw *hw = &pf->hw; 3091 u32 pf_intr_start_offset; 3092 u32 val; 3093 3094 /* Disable anti-spoof detection interrupt to prevent spurious event 3095 * interrupts during a function reset. Anti-spoof functionally is 3096 * still supported. 3097 */ 3098 val = rd32(hw, GL_MDCK_TX_TDPU); 3099 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 3100 wr32(hw, GL_MDCK_TX_TDPU, val); 3101 3102 /* clear things first */ 3103 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 3104 rd32(hw, PFINT_OICR); /* read to clear */ 3105 3106 val = (PFINT_OICR_ECC_ERR_M | 3107 PFINT_OICR_MAL_DETECT_M | 3108 PFINT_OICR_GRST_M | 3109 PFINT_OICR_PCI_EXCEPTION_M | 3110 PFINT_OICR_VFLR_M | 3111 PFINT_OICR_HMC_ERR_M | 3112 PFINT_OICR_PE_PUSH_M | 3113 PFINT_OICR_PE_CRITERR_M); 3114 3115 wr32(hw, PFINT_OICR_ENA, val); 3116 3117 /* SW_ITR_IDX = 0, but don't change INTENA */ 3118 wr32(hw, GLINT_DYN_CTL(pf->oicr_irq.index), 3119 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 3120 3121 if (!pf->hw.dev_caps.ts_dev_info.ts_ll_int_read) 3122 return; 3123 pf_intr_start_offset = rd32(hw, PFINT_ALLOC) & PFINT_ALLOC_FIRST; 3124 wr32(hw, GLINT_DYN_CTL(pf->ll_ts_irq.index + pf_intr_start_offset), 3125 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 3126} 3127 3128/** 3129 * ice_ll_ts_intr - ll_ts interrupt handler 3130 * @irq: interrupt number 3131 * @data: pointer to a q_vector 3132 */ 3133static irqreturn_t ice_ll_ts_intr(int __always_unused irq, void *data) 3134{ 3135 struct ice_pf *pf = data; 3136 u32 pf_intr_start_offset; 3137 struct ice_ptp_tx *tx; 3138 unsigned long flags; 3139 struct ice_hw *hw; 3140 u32 val; 3141 u8 idx; 3142 3143 hw = &pf->hw; 3144 tx = &pf->ptp.port.tx; 3145 spin_lock_irqsave(&tx->lock, flags); 3146 if (tx->init) { 3147 ice_ptp_complete_tx_single_tstamp(tx); 3148 3149 idx = find_next_bit_wrap(tx->in_use, tx->len, 3150 tx->last_ll_ts_idx_read + 1); 3151 if (idx != tx->len) 3152 ice_ptp_req_tx_single_tstamp(tx, idx); 3153 } 3154 spin_unlock_irqrestore(&tx->lock, flags); 3155 3156 val = GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M | 3157 (ICE_ITR_NONE << GLINT_DYN_CTL_ITR_INDX_S); 3158 pf_intr_start_offset = rd32(hw, PFINT_ALLOC) & PFINT_ALLOC_FIRST; 3159 wr32(hw, GLINT_DYN_CTL(pf->ll_ts_irq.index + pf_intr_start_offset), 3160 val); 3161 3162 return IRQ_HANDLED; 3163} 3164 3165/** 3166 * ice_misc_intr - misc interrupt handler 3167 * @irq: interrupt number 3168 * @data: pointer to a q_vector 3169 */ 3170static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 3171{ 3172 struct ice_pf *pf = (struct ice_pf *)data; 3173 irqreturn_t ret = IRQ_HANDLED; 3174 struct ice_hw *hw = &pf->hw; 3175 struct device *dev; 3176 u32 oicr, ena_mask; 3177 3178 dev = ice_pf_to_dev(pf); 3179 set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 3180 set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 3181 set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 3182 3183 oicr = rd32(hw, PFINT_OICR); 3184 ena_mask = rd32(hw, PFINT_OICR_ENA); 3185 3186 if (oicr & PFINT_OICR_SWINT_M) { 3187 ena_mask &= ~PFINT_OICR_SWINT_M; 3188 pf->sw_int_count++; 3189 } 3190 3191 if (oicr & PFINT_OICR_MAL_DETECT_M) { 3192 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 3193 set_bit(ICE_MDD_EVENT_PENDING, pf->state); 3194 } 3195 if (oicr & PFINT_OICR_VFLR_M) { 3196 /* disable any further VFLR event notifications */ 3197 if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) { 3198 u32 reg = rd32(hw, PFINT_OICR_ENA); 3199 3200 reg &= ~PFINT_OICR_VFLR_M; 3201 wr32(hw, PFINT_OICR_ENA, reg); 3202 } else { 3203 ena_mask &= ~PFINT_OICR_VFLR_M; 3204 set_bit(ICE_VFLR_EVENT_PENDING, pf->state); 3205 } 3206 } 3207 3208 if (oicr & PFINT_OICR_GRST_M) { 3209 u32 reset; 3210 3211 /* we have a reset warning */ 3212 ena_mask &= ~PFINT_OICR_GRST_M; 3213 reset = FIELD_GET(GLGEN_RSTAT_RESET_TYPE_M, 3214 rd32(hw, GLGEN_RSTAT)); 3215 3216 if (reset == ICE_RESET_CORER) 3217 pf->corer_count++; 3218 else if (reset == ICE_RESET_GLOBR) 3219 pf->globr_count++; 3220 else if (reset == ICE_RESET_EMPR) 3221 pf->empr_count++; 3222 else 3223 dev_dbg(dev, "Invalid reset type %d\n", reset); 3224 3225 /* If a reset cycle isn't already in progress, we set a bit in 3226 * pf->state so that the service task can start a reset/rebuild. 3227 */ 3228 if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) { 3229 if (reset == ICE_RESET_CORER) 3230 set_bit(ICE_CORER_RECV, pf->state); 3231 else if (reset == ICE_RESET_GLOBR) 3232 set_bit(ICE_GLOBR_RECV, pf->state); 3233 else 3234 set_bit(ICE_EMPR_RECV, pf->state); 3235 3236 /* There are couple of different bits at play here. 3237 * hw->reset_ongoing indicates whether the hardware is 3238 * in reset. This is set to true when a reset interrupt 3239 * is received and set back to false after the driver 3240 * has determined that the hardware is out of reset. 3241 * 3242 * ICE_RESET_OICR_RECV in pf->state indicates 3243 * that a post reset rebuild is required before the 3244 * driver is operational again. This is set above. 3245 * 3246 * As this is the start of the reset/rebuild cycle, set 3247 * both to indicate that. 3248 */ 3249 hw->reset_ongoing = true; 3250 } 3251 } 3252 3253 if (oicr & PFINT_OICR_TSYN_TX_M) { 3254 ena_mask &= ~PFINT_OICR_TSYN_TX_M; 3255 3256 ret = ice_ptp_ts_irq(pf); 3257 } 3258 3259 if (oicr & PFINT_OICR_TSYN_EVNT_M) { 3260 u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; 3261 u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx)); 3262 3263 ena_mask &= ~PFINT_OICR_TSYN_EVNT_M; 3264 3265 if (ice_pf_src_tmr_owned(pf)) { 3266 /* Save EVENTs from GLTSYN register */ 3267 pf->ptp.ext_ts_irq |= gltsyn_stat & 3268 (GLTSYN_STAT_EVENT0_M | 3269 GLTSYN_STAT_EVENT1_M | 3270 GLTSYN_STAT_EVENT2_M); 3271 3272 ice_ptp_extts_event(pf); 3273 } 3274 } 3275 3276#define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M) 3277 if (oicr & ICE_AUX_CRIT_ERR) { 3278 pf->oicr_err_reg |= oicr; 3279 set_bit(ICE_AUX_ERR_PENDING, pf->state); 3280 ena_mask &= ~ICE_AUX_CRIT_ERR; 3281 } 3282 3283 /* Report any remaining unexpected interrupts */ 3284 oicr &= ena_mask; 3285 if (oicr) { 3286 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 3287 /* If a critical error is pending there is no choice but to 3288 * reset the device. 3289 */ 3290 if (oicr & (PFINT_OICR_PCI_EXCEPTION_M | 3291 PFINT_OICR_ECC_ERR_M)) { 3292 set_bit(ICE_PFR_REQ, pf->state); 3293 } 3294 } 3295 ice_service_task_schedule(pf); 3296 if (ret == IRQ_HANDLED) 3297 ice_irq_dynamic_ena(hw, NULL, NULL); 3298 3299 return ret; 3300} 3301 3302/** 3303 * ice_misc_intr_thread_fn - misc interrupt thread function 3304 * @irq: interrupt number 3305 * @data: pointer to a q_vector 3306 */ 3307static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data) 3308{ 3309 struct ice_pf *pf = data; 3310 struct ice_hw *hw; 3311 3312 hw = &pf->hw; 3313 3314 if (ice_is_reset_in_progress(pf->state)) 3315 goto skip_irq; 3316 3317 if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) 3318 ice_ptp_process_ts(pf); 3319 3320skip_irq: 3321 ice_irq_dynamic_ena(hw, NULL, NULL); 3322 ice_flush(hw); 3323 3324 if (ice_ptp_tx_tstamps_pending(pf)) { 3325 /* If any new Tx timestamps happened while in interrupt, 3326 * re-arm the interrupt to trigger it again. 3327 */ 3328 wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M); 3329 ice_flush(hw); 3330 } 3331 3332 return IRQ_HANDLED; 3333} 3334 3335/** 3336 * ice_dis_ctrlq_interrupts - disable control queue interrupts 3337 * @hw: pointer to HW structure 3338 */ 3339static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 3340{ 3341 /* disable Admin queue Interrupt causes */ 3342 wr32(hw, PFINT_FW_CTL, 3343 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 3344 3345 /* disable Mailbox queue Interrupt causes */ 3346 wr32(hw, PFINT_MBX_CTL, 3347 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 3348 3349 wr32(hw, PFINT_SB_CTL, 3350 rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M); 3351 3352 /* disable Control queue Interrupt causes */ 3353 wr32(hw, PFINT_OICR_CTL, 3354 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 3355 3356 ice_flush(hw); 3357} 3358 3359/** 3360 * ice_free_irq_msix_ll_ts- Unroll ll_ts vector setup 3361 * @pf: board private structure 3362 */ 3363static void ice_free_irq_msix_ll_ts(struct ice_pf *pf) 3364{ 3365 int irq_num = pf->ll_ts_irq.virq; 3366 3367 synchronize_irq(irq_num); 3368 devm_free_irq(ice_pf_to_dev(pf), irq_num, pf); 3369 3370 ice_free_irq(pf, pf->ll_ts_irq); 3371} 3372 3373/** 3374 * ice_free_irq_msix_misc - Unroll misc vector setup 3375 * @pf: board private structure 3376 */ 3377static void ice_free_irq_msix_misc(struct ice_pf *pf) 3378{ 3379 int misc_irq_num = pf->oicr_irq.virq; 3380 struct ice_hw *hw = &pf->hw; 3381 3382 ice_dis_ctrlq_interrupts(hw); 3383 3384 /* disable OICR interrupt */ 3385 wr32(hw, PFINT_OICR_ENA, 0); 3386 ice_flush(hw); 3387 3388 synchronize_irq(misc_irq_num); 3389 devm_free_irq(ice_pf_to_dev(pf), misc_irq_num, pf); 3390 3391 ice_free_irq(pf, pf->oicr_irq); 3392 if (pf->hw.dev_caps.ts_dev_info.ts_ll_int_read) 3393 ice_free_irq_msix_ll_ts(pf); 3394} 3395 3396/** 3397 * ice_ena_ctrlq_interrupts - enable control queue interrupts 3398 * @hw: pointer to HW structure 3399 * @reg_idx: HW vector index to associate the control queue interrupts with 3400 */ 3401static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 3402{ 3403 u32 val; 3404 3405 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 3406 PFINT_OICR_CTL_CAUSE_ENA_M); 3407 wr32(hw, PFINT_OICR_CTL, val); 3408 3409 /* enable Admin queue Interrupt causes */ 3410 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 3411 PFINT_FW_CTL_CAUSE_ENA_M); 3412 wr32(hw, PFINT_FW_CTL, val); 3413 3414 /* enable Mailbox queue Interrupt causes */ 3415 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 3416 PFINT_MBX_CTL_CAUSE_ENA_M); 3417 wr32(hw, PFINT_MBX_CTL, val); 3418 3419 if (!hw->dev_caps.ts_dev_info.ts_ll_int_read) { 3420 /* enable Sideband queue Interrupt causes */ 3421 val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) | 3422 PFINT_SB_CTL_CAUSE_ENA_M); 3423 wr32(hw, PFINT_SB_CTL, val); 3424 } 3425 3426 ice_flush(hw); 3427} 3428 3429/** 3430 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 3431 * @pf: board private structure 3432 * 3433 * This sets up the handler for MSIX 0, which is used to manage the 3434 * non-queue interrupts, e.g. AdminQ and errors. This is not used 3435 * when in MSI or Legacy interrupt mode. 3436 */ 3437static int ice_req_irq_msix_misc(struct ice_pf *pf) 3438{ 3439 struct device *dev = ice_pf_to_dev(pf); 3440 struct ice_hw *hw = &pf->hw; 3441 u32 pf_intr_start_offset; 3442 struct msi_map irq; 3443 int err = 0; 3444 3445 if (!pf->int_name[0]) 3446 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 3447 dev_driver_string(dev), dev_name(dev)); 3448 3449 if (!pf->int_name_ll_ts[0]) 3450 snprintf(pf->int_name_ll_ts, sizeof(pf->int_name_ll_ts) - 1, 3451 "%s-%s:ll_ts", dev_driver_string(dev), dev_name(dev)); 3452 /* Do not request IRQ but do enable OICR interrupt since settings are 3453 * lost during reset. Note that this function is called only during 3454 * rebuild path and not while reset is in progress. 3455 */ 3456 if (ice_is_reset_in_progress(pf->state)) 3457 goto skip_req_irq; 3458 3459 /* reserve one vector in irq_tracker for misc interrupts */ 3460 irq = ice_alloc_irq(pf, false); 3461 if (irq.index < 0) 3462 return irq.index; 3463 3464 pf->oicr_irq = irq; 3465 err = devm_request_threaded_irq(dev, pf->oicr_irq.virq, ice_misc_intr, 3466 ice_misc_intr_thread_fn, 0, 3467 pf->int_name, pf); 3468 if (err) { 3469 dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n", 3470 pf->int_name, err); 3471 ice_free_irq(pf, pf->oicr_irq); 3472 return err; 3473 } 3474 3475 /* reserve one vector in irq_tracker for ll_ts interrupt */ 3476 if (!pf->hw.dev_caps.ts_dev_info.ts_ll_int_read) 3477 goto skip_req_irq; 3478 3479 irq = ice_alloc_irq(pf, false); 3480 if (irq.index < 0) 3481 return irq.index; 3482 3483 pf->ll_ts_irq = irq; 3484 err = devm_request_irq(dev, pf->ll_ts_irq.virq, ice_ll_ts_intr, 0, 3485 pf->int_name_ll_ts, pf); 3486 if (err) { 3487 dev_err(dev, "devm_request_irq for %s failed: %d\n", 3488 pf->int_name_ll_ts, err); 3489 ice_free_irq(pf, pf->ll_ts_irq); 3490 return err; 3491 } 3492 3493skip_req_irq: 3494 ice_ena_misc_vector(pf); 3495 3496 ice_ena_ctrlq_interrupts(hw, pf->oicr_irq.index); 3497 /* This enables LL TS interrupt */ 3498 pf_intr_start_offset = rd32(hw, PFINT_ALLOC) & PFINT_ALLOC_FIRST; 3499 if (pf->hw.dev_caps.ts_dev_info.ts_ll_int_read) 3500 wr32(hw, PFINT_SB_CTL, 3501 ((pf->ll_ts_irq.index + pf_intr_start_offset) & 3502 PFINT_SB_CTL_MSIX_INDX_M) | PFINT_SB_CTL_CAUSE_ENA_M); 3503 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_irq.index), 3504 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 3505 3506 ice_flush(hw); 3507 ice_irq_dynamic_ena(hw, NULL, NULL); 3508 3509 return 0; 3510} 3511 3512/** 3513 * ice_set_ops - set netdev and ethtools ops for the given netdev 3514 * @vsi: the VSI associated with the new netdev 3515 */ 3516static void ice_set_ops(struct ice_vsi *vsi) 3517{ 3518 struct net_device *netdev = vsi->netdev; 3519 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3520 3521 if (ice_is_safe_mode(pf)) { 3522 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 3523 ice_set_ethtool_safe_mode_ops(netdev); 3524 return; 3525 } 3526 3527 netdev->netdev_ops = &ice_netdev_ops; 3528 netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; 3529 netdev->xdp_metadata_ops = &ice_xdp_md_ops; 3530 ice_set_ethtool_ops(netdev); 3531 3532 if (vsi->type != ICE_VSI_PF) 3533 return; 3534 3535 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 3536 NETDEV_XDP_ACT_XSK_ZEROCOPY | 3537 NETDEV_XDP_ACT_RX_SG; 3538 netdev->xdp_zc_max_segs = ICE_MAX_BUF_TXD; 3539} 3540 3541/** 3542 * ice_set_netdev_features - set features for the given netdev 3543 * @netdev: netdev instance 3544 */ 3545void ice_set_netdev_features(struct net_device *netdev) 3546{ 3547 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3548 bool is_dvm_ena = ice_is_dvm_ena(&pf->hw); 3549 netdev_features_t csumo_features; 3550 netdev_features_t vlano_features; 3551 netdev_features_t dflt_features; 3552 netdev_features_t tso_features; 3553 3554 if (ice_is_safe_mode(pf)) { 3555 /* safe mode */ 3556 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 3557 netdev->hw_features = netdev->features; 3558 return; 3559 } 3560 3561 dflt_features = NETIF_F_SG | 3562 NETIF_F_HIGHDMA | 3563 NETIF_F_NTUPLE | 3564 NETIF_F_RXHASH; 3565 3566 csumo_features = NETIF_F_RXCSUM | 3567 NETIF_F_IP_CSUM | 3568 NETIF_F_SCTP_CRC | 3569 NETIF_F_IPV6_CSUM; 3570 3571 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 3572 NETIF_F_HW_VLAN_CTAG_TX | 3573 NETIF_F_HW_VLAN_CTAG_RX; 3574 3575 /* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */ 3576 if (is_dvm_ena) 3577 vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER; 3578 3579 tso_features = NETIF_F_TSO | 3580 NETIF_F_TSO_ECN | 3581 NETIF_F_TSO6 | 3582 NETIF_F_GSO_GRE | 3583 NETIF_F_GSO_UDP_TUNNEL | 3584 NETIF_F_GSO_GRE_CSUM | 3585 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3586 NETIF_F_GSO_PARTIAL | 3587 NETIF_F_GSO_IPXIP4 | 3588 NETIF_F_GSO_IPXIP6 | 3589 NETIF_F_GSO_UDP_L4; 3590 3591 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 3592 NETIF_F_GSO_GRE_CSUM; 3593 /* set features that user can change */ 3594 netdev->hw_features = dflt_features | csumo_features | 3595 vlano_features | tso_features; 3596 3597 /* add support for HW_CSUM on packets with MPLS header */ 3598 netdev->mpls_features = NETIF_F_HW_CSUM | 3599 NETIF_F_TSO | 3600 NETIF_F_TSO6; 3601 3602 /* enable features */ 3603 netdev->features |= netdev->hw_features; 3604 3605 netdev->hw_features |= NETIF_F_HW_TC; 3606 netdev->hw_features |= NETIF_F_LOOPBACK; 3607 3608 /* encap and VLAN devices inherit default, csumo and tso features */ 3609 netdev->hw_enc_features |= dflt_features | csumo_features | 3610 tso_features; 3611 netdev->vlan_features |= dflt_features | csumo_features | 3612 tso_features; 3613 3614 /* advertise support but don't enable by default since only one type of 3615 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one 3616 * type turns on the other has to be turned off. This is enforced by the 3617 * ice_fix_features() ndo callback. 3618 */ 3619 if (is_dvm_ena) 3620 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX | 3621 NETIF_F_HW_VLAN_STAG_TX; 3622 3623 /* Leave CRC / FCS stripping enabled by default, but allow the value to 3624 * be changed at runtime 3625 */ 3626 netdev->hw_features |= NETIF_F_RXFCS; 3627 3628 /* Allow core to manage IRQs affinity */ 3629 netif_set_affinity_auto(netdev); 3630 3631 /* Mutual exclusivity for TSO and GCS is enforced by the set features 3632 * ndo callback. 3633 */ 3634 if (ice_is_feature_supported(pf, ICE_F_GCS)) 3635 netdev->hw_features |= NETIF_F_HW_CSUM; 3636 3637 netif_set_tso_max_size(netdev, ICE_MAX_TSO_SIZE); 3638} 3639 3640/** 3641 * ice_fill_rss_lut - Fill the RSS lookup table with default values 3642 * @lut: Lookup table 3643 * @rss_table_size: Lookup table size 3644 * @rss_size: Range of queue number for hashing 3645 */ 3646void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 3647{ 3648 u16 i; 3649 3650 for (i = 0; i < rss_table_size; i++) 3651 lut[i] = i % rss_size; 3652} 3653 3654/** 3655 * ice_pf_vsi_setup - Set up a PF VSI 3656 * @pf: board private structure 3657 * @pi: pointer to the port_info instance 3658 * 3659 * Returns pointer to the successfully allocated VSI software struct 3660 * on success, otherwise returns NULL on failure. 3661 */ 3662static struct ice_vsi * 3663ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3664{ 3665 struct ice_vsi_cfg_params params = {}; 3666 3667 params.type = ICE_VSI_PF; 3668 params.port_info = pi; 3669 params.flags = ICE_VSI_FLAG_INIT; 3670 3671 return ice_vsi_setup(pf, ¶ms); 3672} 3673 3674static struct ice_vsi * 3675ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, 3676 struct ice_channel *ch) 3677{ 3678 struct ice_vsi_cfg_params params = {}; 3679 3680 params.type = ICE_VSI_CHNL; 3681 params.port_info = pi; 3682 params.ch = ch; 3683 params.flags = ICE_VSI_FLAG_INIT; 3684 3685 return ice_vsi_setup(pf, ¶ms); 3686} 3687 3688/** 3689 * ice_ctrl_vsi_setup - Set up a control VSI 3690 * @pf: board private structure 3691 * @pi: pointer to the port_info instance 3692 * 3693 * Returns pointer to the successfully allocated VSI software struct 3694 * on success, otherwise returns NULL on failure. 3695 */ 3696static struct ice_vsi * 3697ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3698{ 3699 struct ice_vsi_cfg_params params = {}; 3700 3701 params.type = ICE_VSI_CTRL; 3702 params.port_info = pi; 3703 params.flags = ICE_VSI_FLAG_INIT; 3704 3705 return ice_vsi_setup(pf, ¶ms); 3706} 3707 3708/** 3709 * ice_lb_vsi_setup - Set up a loopback VSI 3710 * @pf: board private structure 3711 * @pi: pointer to the port_info instance 3712 * 3713 * Returns pointer to the successfully allocated VSI software struct 3714 * on success, otherwise returns NULL on failure. 3715 */ 3716struct ice_vsi * 3717ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3718{ 3719 struct ice_vsi_cfg_params params = {}; 3720 3721 params.type = ICE_VSI_LB; 3722 params.port_info = pi; 3723 params.flags = ICE_VSI_FLAG_INIT; 3724 3725 return ice_vsi_setup(pf, ¶ms); 3726} 3727 3728/** 3729 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 3730 * @netdev: network interface to be adjusted 3731 * @proto: VLAN TPID 3732 * @vid: VLAN ID to be added 3733 * 3734 * net_device_ops implementation for adding VLAN IDs 3735 */ 3736int ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 3737{ 3738 struct ice_netdev_priv *np = netdev_priv(netdev); 3739 struct ice_vsi_vlan_ops *vlan_ops; 3740 struct ice_vsi *vsi = np->vsi; 3741 struct ice_vlan vlan; 3742 int ret; 3743 3744 /* VLAN 0 is added by default during load/reset */ 3745 if (!vid) 3746 return 0; 3747 3748 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3749 usleep_range(1000, 2000); 3750 3751 /* Add multicast promisc rule for the VLAN ID to be added if 3752 * all-multicast is currently enabled. 3753 */ 3754 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3755 ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3756 ICE_MCAST_VLAN_PROMISC_BITS, 3757 vid); 3758 if (ret) 3759 goto finish; 3760 } 3761 3762 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3763 3764 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 3765 * packets aren't pruned by the device's internal switch on Rx 3766 */ 3767 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3768 ret = vlan_ops->add_vlan(vsi, &vlan); 3769 if (ret) 3770 goto finish; 3771 3772 /* If all-multicast is currently enabled and this VLAN ID is only one 3773 * besides VLAN-0 we have to update look-up type of multicast promisc 3774 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN. 3775 */ 3776 if ((vsi->current_netdev_flags & IFF_ALLMULTI) && 3777 ice_vsi_num_non_zero_vlans(vsi) == 1) { 3778 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3779 ICE_MCAST_PROMISC_BITS, 0); 3780 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3781 ICE_MCAST_VLAN_PROMISC_BITS, 0); 3782 } 3783 3784finish: 3785 clear_bit(ICE_CFG_BUSY, vsi->state); 3786 3787 return ret; 3788} 3789 3790/** 3791 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 3792 * @netdev: network interface to be adjusted 3793 * @proto: VLAN TPID 3794 * @vid: VLAN ID to be removed 3795 * 3796 * net_device_ops implementation for removing VLAN IDs 3797 */ 3798int ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 3799{ 3800 struct ice_netdev_priv *np = netdev_priv(netdev); 3801 struct ice_vsi_vlan_ops *vlan_ops; 3802 struct ice_vsi *vsi = np->vsi; 3803 struct ice_vlan vlan; 3804 int ret; 3805 3806 /* don't allow removal of VLAN 0 */ 3807 if (!vid) 3808 return 0; 3809 3810 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3811 usleep_range(1000, 2000); 3812 3813 ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3814 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3815 if (ret) { 3816 netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n", 3817 vsi->vsi_num); 3818 vsi->current_netdev_flags |= IFF_ALLMULTI; 3819 } 3820 3821 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3822 3823 /* Make sure VLAN delete is successful before updating VLAN 3824 * information 3825 */ 3826 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3827 ret = vlan_ops->del_vlan(vsi, &vlan); 3828 if (ret) 3829 goto finish; 3830 3831 /* Remove multicast promisc rule for the removed VLAN ID if 3832 * all-multicast is enabled. 3833 */ 3834 if (vsi->current_netdev_flags & IFF_ALLMULTI) 3835 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3836 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3837 3838 if (!ice_vsi_has_non_zero_vlans(vsi)) { 3839 /* Update look-up type of multicast promisc rule for VLAN 0 3840 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when 3841 * all-multicast is enabled and VLAN 0 is the only VLAN rule. 3842 */ 3843 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3844 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3845 ICE_MCAST_VLAN_PROMISC_BITS, 3846 0); 3847 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3848 ICE_MCAST_PROMISC_BITS, 0); 3849 } 3850 } 3851 3852finish: 3853 clear_bit(ICE_CFG_BUSY, vsi->state); 3854 3855 return ret; 3856} 3857 3858/** 3859 * ice_rep_indr_tc_block_unbind 3860 * @cb_priv: indirection block private data 3861 */ 3862static void ice_rep_indr_tc_block_unbind(void *cb_priv) 3863{ 3864 struct ice_indr_block_priv *indr_priv = cb_priv; 3865 3866 list_del(&indr_priv->list); 3867 kfree(indr_priv); 3868} 3869 3870/** 3871 * ice_tc_indir_block_unregister - Unregister TC indirect block notifications 3872 * @vsi: VSI struct which has the netdev 3873 */ 3874static void ice_tc_indir_block_unregister(struct ice_vsi *vsi) 3875{ 3876 struct ice_netdev_priv *np = netdev_priv(vsi->netdev); 3877 3878 flow_indr_dev_unregister(ice_indr_setup_tc_cb, np, 3879 ice_rep_indr_tc_block_unbind); 3880} 3881 3882/** 3883 * ice_tc_indir_block_register - Register TC indirect block notifications 3884 * @vsi: VSI struct which has the netdev 3885 * 3886 * Returns 0 on success, negative value on failure 3887 */ 3888static int ice_tc_indir_block_register(struct ice_vsi *vsi) 3889{ 3890 struct ice_netdev_priv *np; 3891 3892 if (!vsi || !vsi->netdev) 3893 return -EINVAL; 3894 3895 np = netdev_priv(vsi->netdev); 3896 3897 INIT_LIST_HEAD(&np->tc_indr_block_priv_list); 3898 return flow_indr_dev_register(ice_indr_setup_tc_cb, np); 3899} 3900 3901/** 3902 * ice_get_avail_q_count - Get count of queues in use 3903 * @pf_qmap: bitmap to get queue use count from 3904 * @lock: pointer to a mutex that protects access to pf_qmap 3905 * @size: size of the bitmap 3906 */ 3907static u16 3908ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 3909{ 3910 unsigned long bit; 3911 u16 count = 0; 3912 3913 mutex_lock(lock); 3914 for_each_clear_bit(bit, pf_qmap, size) 3915 count++; 3916 mutex_unlock(lock); 3917 3918 return count; 3919} 3920 3921/** 3922 * ice_get_avail_txq_count - Get count of Tx queues in use 3923 * @pf: pointer to an ice_pf instance 3924 */ 3925u16 ice_get_avail_txq_count(struct ice_pf *pf) 3926{ 3927 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3928 pf->max_pf_txqs); 3929} 3930 3931/** 3932 * ice_get_avail_rxq_count - Get count of Rx queues in use 3933 * @pf: pointer to an ice_pf instance 3934 */ 3935u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3936{ 3937 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3938 pf->max_pf_rxqs); 3939} 3940 3941/** 3942 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3943 * @pf: board private structure to initialize 3944 */ 3945void ice_deinit_pf(struct ice_pf *pf) 3946{ 3947 /* note that we unroll also on ice_init_pf() failure here */ 3948 3949 mutex_destroy(&pf->lag_mutex); 3950 mutex_destroy(&pf->adev_mutex); 3951 mutex_destroy(&pf->sw_mutex); 3952 mutex_destroy(&pf->tc_mutex); 3953 mutex_destroy(&pf->avail_q_mutex); 3954 mutex_destroy(&pf->vfs.table_lock); 3955 3956 if (pf->avail_txqs) { 3957 bitmap_free(pf->avail_txqs); 3958 pf->avail_txqs = NULL; 3959 } 3960 3961 if (pf->avail_rxqs) { 3962 bitmap_free(pf->avail_rxqs); 3963 pf->avail_rxqs = NULL; 3964 } 3965 3966 if (pf->txtime_txqs) { 3967 bitmap_free(pf->txtime_txqs); 3968 pf->txtime_txqs = NULL; 3969 } 3970 3971 if (pf->ptp.clock) 3972 ptp_clock_unregister(pf->ptp.clock); 3973 3974 if (!xa_empty(&pf->irq_tracker.entries)) 3975 ice_free_irq_msix_misc(pf); 3976 3977 xa_destroy(&pf->dyn_ports); 3978 xa_destroy(&pf->sf_nums); 3979} 3980 3981/** 3982 * ice_set_pf_caps - set PFs capability flags 3983 * @pf: pointer to the PF instance 3984 */ 3985static void ice_set_pf_caps(struct ice_pf *pf) 3986{ 3987 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3988 3989 clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3990 if (func_caps->common_cap.rdma) 3991 set_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3992 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3993 if (func_caps->common_cap.dcb) 3994 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3995 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3996 if (func_caps->common_cap.sr_iov_1_1) { 3997 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3998 pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs, 3999 ICE_MAX_SRIOV_VFS); 4000 } 4001 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 4002 if (func_caps->common_cap.rss_table_size) 4003 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 4004 4005 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 4006 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 4007 u16 unused; 4008 4009 /* ctrl_vsi_idx will be set to a valid value when flow director 4010 * is setup by ice_init_fdir 4011 */ 4012 pf->ctrl_vsi_idx = ICE_NO_VSI; 4013 set_bit(ICE_FLAG_FD_ENA, pf->flags); 4014 /* force guaranteed filter pool for PF */ 4015 ice_alloc_fd_guar_item(&pf->hw, &unused, 4016 func_caps->fd_fltr_guar); 4017 /* force shared filter pool for PF */ 4018 ice_alloc_fd_shrd_item(&pf->hw, &unused, 4019 func_caps->fd_fltr_best_effort); 4020 } 4021 4022 clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 4023 if (func_caps->common_cap.ieee_1588) 4024 set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 4025 4026 pf->max_pf_txqs = func_caps->common_cap.num_txq; 4027 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 4028} 4029 4030void ice_start_service_task(struct ice_pf *pf) 4031{ 4032 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 4033 pf->serv_tmr_period = HZ; 4034 INIT_WORK(&pf->serv_task, ice_service_task); 4035 clear_bit(ICE_SERVICE_SCHED, pf->state); 4036} 4037 4038/** 4039 * ice_init_pf - Initialize general software structures (struct ice_pf) 4040 * @pf: board private structure to initialize 4041 * Return: 0 on success, negative errno otherwise. 4042 */ 4043int ice_init_pf(struct ice_pf *pf) 4044{ 4045 struct udp_tunnel_nic_info *udp_tunnel_nic = &pf->hw.udp_tunnel_nic; 4046 struct device *dev = ice_pf_to_dev(pf); 4047 struct ice_hw *hw = &pf->hw; 4048 int err = -ENOMEM; 4049 4050 mutex_init(&pf->sw_mutex); 4051 mutex_init(&pf->tc_mutex); 4052 mutex_init(&pf->adev_mutex); 4053 mutex_init(&pf->lag_mutex); 4054 4055 INIT_HLIST_HEAD(&pf->aq_wait_list); 4056 spin_lock_init(&pf->aq_wait_lock); 4057 init_waitqueue_head(&pf->aq_wait_queue); 4058 4059 init_waitqueue_head(&pf->reset_wait_queue); 4060 4061 mutex_init(&pf->avail_q_mutex); 4062 4063 mutex_init(&pf->vfs.table_lock); 4064 hash_init(pf->vfs.table); 4065 if (ice_is_feature_supported(pf, ICE_F_MBX_LIMIT)) 4066 wr32(&pf->hw, E830_MBX_PF_IN_FLIGHT_VF_MSGS_THRESH, 4067 ICE_MBX_OVERFLOW_WATERMARK); 4068 else 4069 ice_mbx_init_snapshot(&pf->hw); 4070 4071 xa_init(&pf->dyn_ports); 4072 xa_init(&pf->sf_nums); 4073 4074 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 4075 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 4076 pf->txtime_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 4077 if (!pf->avail_txqs || !pf->avail_rxqs || !pf->txtime_txqs) 4078 goto undo_init; 4079 4080 udp_tunnel_nic->set_port = ice_udp_tunnel_set_port; 4081 udp_tunnel_nic->unset_port = ice_udp_tunnel_unset_port; 4082 udp_tunnel_nic->shared = &hw->udp_tunnel_shared; 4083 udp_tunnel_nic->tables[0].n_entries = hw->tnl.valid_count[TNL_VXLAN]; 4084 udp_tunnel_nic->tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 4085 udp_tunnel_nic->tables[1].n_entries = hw->tnl.valid_count[TNL_GENEVE]; 4086 udp_tunnel_nic->tables[1].tunnel_types = UDP_TUNNEL_TYPE_GENEVE; 4087 4088 /* In case of MSIX we are going to setup the misc vector right here 4089 * to handle admin queue events etc. In case of legacy and MSI 4090 * the misc functionality and queue processing is combined in 4091 * the same vector and that gets setup at open. 4092 */ 4093 err = ice_req_irq_msix_misc(pf); 4094 if (err) { 4095 dev_err(dev, "setup of misc vector failed: %d\n", err); 4096 goto undo_init; 4097 } 4098 4099 return 0; 4100undo_init: 4101 /* deinit handles half-initialized pf just fine */ 4102 ice_deinit_pf(pf); 4103 return err; 4104} 4105 4106/** 4107 * ice_is_wol_supported - check if WoL is supported 4108 * @hw: pointer to hardware info 4109 * 4110 * Check if WoL is supported based on the HW configuration. 4111 * Returns true if NVM supports and enables WoL for this port, false otherwise 4112 */ 4113bool ice_is_wol_supported(struct ice_hw *hw) 4114{ 4115 u16 wol_ctrl; 4116 4117 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 4118 * word) indicates WoL is not supported on the corresponding PF ID. 4119 */ 4120 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 4121 return false; 4122 4123 return !(BIT(hw->port_info->lport) & wol_ctrl); 4124} 4125 4126/** 4127 * ice_vsi_recfg_qs - Change the number of queues on a VSI 4128 * @vsi: VSI being changed 4129 * @new_rx: new number of Rx queues 4130 * @new_tx: new number of Tx queues 4131 * @locked: is adev device_lock held 4132 * 4133 * Only change the number of queues if new_tx, or new_rx is non-0. 4134 * 4135 * Returns 0 on success. 4136 */ 4137int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked) 4138{ 4139 struct ice_pf *pf = vsi->back; 4140 int i, err = 0, timeout = 50; 4141 4142 if (!new_rx && !new_tx) 4143 return -EINVAL; 4144 4145 while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) { 4146 timeout--; 4147 if (!timeout) 4148 return -EBUSY; 4149 usleep_range(1000, 2000); 4150 } 4151 4152 if (new_tx) 4153 vsi->req_txq = (u16)new_tx; 4154 if (new_rx) 4155 vsi->req_rxq = (u16)new_rx; 4156 4157 /* set for the next time the netdev is started */ 4158 if (!netif_running(vsi->netdev)) { 4159 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 4160 if (err) 4161 goto rebuild_err; 4162 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 4163 goto done; 4164 } 4165 4166 ice_vsi_close(vsi); 4167 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 4168 if (err) 4169 goto rebuild_err; 4170 4171 ice_for_each_traffic_class(i) { 4172 if (vsi->tc_cfg.ena_tc & BIT(i)) 4173 netdev_set_tc_queue(vsi->netdev, 4174 vsi->tc_cfg.tc_info[i].netdev_tc, 4175 vsi->tc_cfg.tc_info[i].qcount_tx, 4176 vsi->tc_cfg.tc_info[i].qoffset); 4177 } 4178 ice_pf_dcb_recfg(pf, locked); 4179 ice_vsi_open(vsi); 4180 goto done; 4181 4182rebuild_err: 4183 dev_err(ice_pf_to_dev(pf), "Error during VSI rebuild: %d. Unload and reload the driver.\n", 4184 err); 4185done: 4186 clear_bit(ICE_CFG_BUSY, pf->state); 4187 return err; 4188} 4189 4190/** 4191 * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode 4192 * @pf: PF to configure 4193 * 4194 * No VLAN offloads/filtering are advertised in safe mode so make sure the PF 4195 * VSI can still Tx/Rx VLAN tagged packets. 4196 */ 4197static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf) 4198{ 4199 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4200 struct ice_vsi_ctx *ctxt; 4201 struct ice_hw *hw; 4202 int status; 4203 4204 if (!vsi) 4205 return; 4206 4207 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 4208 if (!ctxt) 4209 return; 4210 4211 hw = &pf->hw; 4212 ctxt->info = vsi->info; 4213 4214 ctxt->info.valid_sections = 4215 cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID | 4216 ICE_AQ_VSI_PROP_SECURITY_VALID | 4217 ICE_AQ_VSI_PROP_SW_VALID); 4218 4219 /* disable VLAN anti-spoof */ 4220 ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << 4221 ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); 4222 4223 /* disable VLAN pruning and keep all other settings */ 4224 ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; 4225 4226 /* allow all VLANs on Tx and don't strip on Rx */ 4227 ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL | 4228 ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING; 4229 4230 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 4231 if (status) { 4232 dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n", 4233 status, libie_aq_str(hw->adminq.sq_last_status)); 4234 } else { 4235 vsi->info.sec_flags = ctxt->info.sec_flags; 4236 vsi->info.sw_flags2 = ctxt->info.sw_flags2; 4237 vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags; 4238 } 4239 4240 kfree(ctxt); 4241} 4242 4243/** 4244 * ice_log_pkg_init - log result of DDP package load 4245 * @hw: pointer to hardware info 4246 * @state: state of package load 4247 */ 4248static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state) 4249{ 4250 struct ice_pf *pf = hw->back; 4251 struct device *dev; 4252 4253 dev = ice_pf_to_dev(pf); 4254 4255 switch (state) { 4256 case ICE_DDP_PKG_SUCCESS: 4257 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 4258 hw->active_pkg_name, 4259 hw->active_pkg_ver.major, 4260 hw->active_pkg_ver.minor, 4261 hw->active_pkg_ver.update, 4262 hw->active_pkg_ver.draft); 4263 break; 4264 case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED: 4265 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 4266 hw->active_pkg_name, 4267 hw->active_pkg_ver.major, 4268 hw->active_pkg_ver.minor, 4269 hw->active_pkg_ver.update, 4270 hw->active_pkg_ver.draft); 4271 break; 4272 case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED: 4273 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 4274 hw->active_pkg_name, 4275 hw->active_pkg_ver.major, 4276 hw->active_pkg_ver.minor, 4277 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4278 break; 4279 case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED: 4280 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 4281 hw->active_pkg_name, 4282 hw->active_pkg_ver.major, 4283 hw->active_pkg_ver.minor, 4284 hw->active_pkg_ver.update, 4285 hw->active_pkg_ver.draft, 4286 hw->pkg_name, 4287 hw->pkg_ver.major, 4288 hw->pkg_ver.minor, 4289 hw->pkg_ver.update, 4290 hw->pkg_ver.draft); 4291 break; 4292 case ICE_DDP_PKG_FW_MISMATCH: 4293 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 4294 break; 4295 case ICE_DDP_PKG_INVALID_FILE: 4296 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 4297 break; 4298 case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH: 4299 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 4300 break; 4301 case ICE_DDP_PKG_FILE_VERSION_TOO_LOW: 4302 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 4303 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4304 break; 4305 case ICE_DDP_PKG_FILE_SIGNATURE_INVALID: 4306 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 4307 break; 4308 case ICE_DDP_PKG_FILE_REVISION_TOO_LOW: 4309 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 4310 break; 4311 case ICE_DDP_PKG_LOAD_ERROR: 4312 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 4313 /* poll for reset to complete */ 4314 if (ice_check_reset(hw)) 4315 dev_err(dev, "Error resetting device. Please reload the driver\n"); 4316 break; 4317 case ICE_DDP_PKG_ERR: 4318 default: 4319 dev_err(dev, "An unknown error occurred when loading the DDP package. Entering Safe Mode.\n"); 4320 break; 4321 } 4322} 4323 4324/** 4325 * ice_load_pkg - load/reload the DDP Package file 4326 * @firmware: firmware structure when firmware requested or NULL for reload 4327 * @pf: pointer to the PF instance 4328 * 4329 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 4330 * initialize HW tables. 4331 */ 4332static void 4333ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 4334{ 4335 enum ice_ddp_state state = ICE_DDP_PKG_ERR; 4336 struct device *dev = ice_pf_to_dev(pf); 4337 struct ice_hw *hw = &pf->hw; 4338 4339 /* Load DDP Package */ 4340 if (firmware && !hw->pkg_copy) { 4341 state = ice_copy_and_init_pkg(hw, firmware->data, 4342 firmware->size); 4343 ice_log_pkg_init(hw, state); 4344 } else if (!firmware && hw->pkg_copy) { 4345 /* Reload package during rebuild after CORER/GLOBR reset */ 4346 state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 4347 ice_log_pkg_init(hw, state); 4348 } else { 4349 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 4350 } 4351 4352 if (!ice_is_init_pkg_successful(state)) { 4353 /* Safe Mode */ 4354 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4355 return; 4356 } 4357 4358 /* Successful download package is the precondition for advanced 4359 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 4360 */ 4361 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4362} 4363 4364/** 4365 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 4366 * @pf: pointer to the PF structure 4367 * 4368 * There is no error returned here because the driver should be able to handle 4369 * 128 Byte cache lines, so we only print a warning in case issues are seen, 4370 * specifically with Tx. 4371 */ 4372static void ice_verify_cacheline_size(struct ice_pf *pf) 4373{ 4374 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 4375 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 4376 ICE_CACHE_LINE_BYTES); 4377} 4378 4379/** 4380 * ice_send_version - update firmware with driver version 4381 * @pf: PF struct 4382 * 4383 * Returns 0 on success, else error code 4384 */ 4385static int ice_send_version(struct ice_pf *pf) 4386{ 4387 struct ice_driver_ver dv; 4388 4389 dv.major_ver = 0xff; 4390 dv.minor_ver = 0xff; 4391 dv.build_ver = 0xff; 4392 dv.subbuild_ver = 0; 4393 strscpy((char *)dv.driver_string, UTS_RELEASE, 4394 sizeof(dv.driver_string)); 4395 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 4396} 4397 4398/** 4399 * ice_init_fdir - Initialize flow director VSI and configuration 4400 * @pf: pointer to the PF instance 4401 * 4402 * returns 0 on success, negative on error 4403 */ 4404static int ice_init_fdir(struct ice_pf *pf) 4405{ 4406 struct device *dev = ice_pf_to_dev(pf); 4407 struct ice_vsi *ctrl_vsi; 4408 int err; 4409 4410 /* Side Band Flow Director needs to have a control VSI. 4411 * Allocate it and store it in the PF. 4412 */ 4413 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 4414 if (!ctrl_vsi) { 4415 dev_dbg(dev, "could not create control VSI\n"); 4416 return -ENOMEM; 4417 } 4418 4419 err = ice_vsi_open_ctrl(ctrl_vsi); 4420 if (err) { 4421 dev_dbg(dev, "could not open control VSI\n"); 4422 goto err_vsi_open; 4423 } 4424 4425 mutex_init(&pf->hw.fdir_fltr_lock); 4426 4427 err = ice_fdir_create_dflt_rules(pf); 4428 if (err) 4429 goto err_fdir_rule; 4430 4431 return 0; 4432 4433err_fdir_rule: 4434 ice_fdir_release_flows(&pf->hw); 4435 ice_vsi_close(ctrl_vsi); 4436err_vsi_open: 4437 ice_vsi_release(ctrl_vsi); 4438 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 4439 pf->vsi[pf->ctrl_vsi_idx] = NULL; 4440 pf->ctrl_vsi_idx = ICE_NO_VSI; 4441 } 4442 return err; 4443} 4444 4445static void ice_deinit_fdir(struct ice_pf *pf) 4446{ 4447 struct ice_vsi *vsi = ice_get_ctrl_vsi(pf); 4448 4449 if (!vsi) 4450 return; 4451 4452 ice_vsi_manage_fdir(vsi, false); 4453 ice_vsi_release(vsi); 4454 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 4455 pf->vsi[pf->ctrl_vsi_idx] = NULL; 4456 pf->ctrl_vsi_idx = ICE_NO_VSI; 4457 } 4458 4459 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 4460} 4461 4462/** 4463 * ice_get_opt_fw_name - return optional firmware file name or NULL 4464 * @pf: pointer to the PF instance 4465 */ 4466static char *ice_get_opt_fw_name(struct ice_pf *pf) 4467{ 4468 /* Optional firmware name same as default with additional dash 4469 * followed by a EUI-64 identifier (PCIe Device Serial Number) 4470 */ 4471 struct pci_dev *pdev = pf->pdev; 4472 char *opt_fw_filename; 4473 u64 dsn; 4474 4475 /* Determine the name of the optional file using the DSN (two 4476 * dwords following the start of the DSN Capability). 4477 */ 4478 dsn = pci_get_dsn(pdev); 4479 if (!dsn) 4480 return NULL; 4481 4482 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 4483 if (!opt_fw_filename) 4484 return NULL; 4485 4486 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 4487 ICE_DDP_PKG_PATH, dsn); 4488 4489 return opt_fw_filename; 4490} 4491 4492/** 4493 * ice_request_fw - Device initialization routine 4494 * @pf: pointer to the PF instance 4495 * @firmware: double pointer to firmware struct 4496 * 4497 * Return: zero when successful, negative values otherwise. 4498 */ 4499static int ice_request_fw(struct ice_pf *pf, const struct firmware **firmware) 4500{ 4501 char *opt_fw_filename = ice_get_opt_fw_name(pf); 4502 struct device *dev = ice_pf_to_dev(pf); 4503 int err = 0; 4504 4505 /* optional device-specific DDP (if present) overrides the default DDP 4506 * package file. kernel logs a debug message if the file doesn't exist, 4507 * and warning messages for other errors. 4508 */ 4509 if (opt_fw_filename) { 4510 err = firmware_request_nowarn(firmware, opt_fw_filename, dev); 4511 kfree(opt_fw_filename); 4512 if (!err) 4513 return err; 4514 } 4515 err = request_firmware(firmware, ICE_DDP_PKG_FILE, dev); 4516 if (err) 4517 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 4518 4519 return err; 4520} 4521 4522/** 4523 * ice_init_tx_topology - performs Tx topology initialization 4524 * @hw: pointer to the hardware structure 4525 * @firmware: pointer to firmware structure 4526 * 4527 * Return: zero when init was successful, negative values otherwise. 4528 */ 4529static int 4530ice_init_tx_topology(struct ice_hw *hw, const struct firmware *firmware) 4531{ 4532 u8 num_tx_sched_layers = hw->num_tx_sched_layers; 4533 struct ice_pf *pf = hw->back; 4534 struct device *dev; 4535 int err; 4536 4537 dev = ice_pf_to_dev(pf); 4538 err = ice_cfg_tx_topo(hw, firmware->data, firmware->size); 4539 if (!err) { 4540 if (hw->num_tx_sched_layers > num_tx_sched_layers) 4541 dev_info(dev, "Tx scheduling layers switching feature disabled\n"); 4542 else 4543 dev_info(dev, "Tx scheduling layers switching feature enabled\n"); 4544 return 0; 4545 } else if (err == -ENODEV) { 4546 /* If we failed to re-initialize the device, we can no longer 4547 * continue loading. 4548 */ 4549 dev_warn(dev, "Failed to initialize hardware after applying Tx scheduling configuration.\n"); 4550 return err; 4551 } else if (err == -EIO) { 4552 dev_info(dev, "DDP package does not support Tx scheduling layers switching feature - please update to the latest DDP package and try again\n"); 4553 return 0; 4554 } else if (err == -EEXIST) { 4555 return 0; 4556 } 4557 4558 /* Do not treat this as a fatal error. */ 4559 dev_info(dev, "Failed to apply Tx scheduling configuration, err %pe\n", 4560 ERR_PTR(err)); 4561 return 0; 4562} 4563 4564/** 4565 * ice_init_supported_rxdids - Initialize supported Rx descriptor IDs 4566 * @hw: pointer to the hardware structure 4567 * @pf: pointer to pf structure 4568 * 4569 * The pf->supported_rxdids bitmap is used to indicate to VFs which descriptor 4570 * formats the PF hardware supports. The exact list of supported RXDIDs 4571 * depends on the loaded DDP package. The IDs can be determined by reading the 4572 * GLFLXP_RXDID_FLAGS register after the DDP package is loaded. 4573 * 4574 * Note that the legacy 32-byte RXDID 0 is always supported but is not listed 4575 * in the DDP package. The 16-byte legacy descriptor is never supported by 4576 * VFs. 4577 */ 4578static void ice_init_supported_rxdids(struct ice_hw *hw, struct ice_pf *pf) 4579{ 4580 pf->supported_rxdids = BIT(ICE_RXDID_LEGACY_1); 4581 4582 for (int i = ICE_RXDID_FLEX_NIC; i < ICE_FLEX_DESC_RXDID_MAX_NUM; i++) { 4583 u32 regval; 4584 4585 regval = rd32(hw, GLFLXP_RXDID_FLAGS(i, 0)); 4586 if ((regval >> GLFLXP_RXDID_FLAGS_FLEXIFLAG_4N_S) 4587 & GLFLXP_RXDID_FLAGS_FLEXIFLAG_4N_M) 4588 pf->supported_rxdids |= BIT(i); 4589 } 4590} 4591 4592/** 4593 * ice_init_ddp_config - DDP related configuration 4594 * @hw: pointer to the hardware structure 4595 * @pf: pointer to pf structure 4596 * 4597 * This function loads DDP file from the disk, then initializes Tx 4598 * topology. At the end DDP package is loaded on the card. 4599 * 4600 * Return: zero when init was successful, negative values otherwise. 4601 */ 4602static int ice_init_ddp_config(struct ice_hw *hw, struct ice_pf *pf) 4603{ 4604 struct device *dev = ice_pf_to_dev(pf); 4605 const struct firmware *firmware = NULL; 4606 int err; 4607 4608 err = ice_request_fw(pf, &firmware); 4609 if (err) { 4610 dev_err(dev, "Fail during requesting FW: %d\n", err); 4611 return err; 4612 } 4613 4614 err = ice_init_tx_topology(hw, firmware); 4615 if (err) { 4616 dev_err(dev, "Fail during initialization of Tx topology: %d\n", 4617 err); 4618 release_firmware(firmware); 4619 return err; 4620 } 4621 4622 /* Download firmware to device */ 4623 ice_load_pkg(firmware, pf); 4624 release_firmware(firmware); 4625 4626 /* Initialize the supported Rx descriptor IDs after loading DDP */ 4627 ice_init_supported_rxdids(hw, pf); 4628 4629 return 0; 4630} 4631 4632/** 4633 * ice_print_wake_reason - show the wake up cause in the log 4634 * @pf: pointer to the PF struct 4635 */ 4636static void ice_print_wake_reason(struct ice_pf *pf) 4637{ 4638 u32 wus = pf->wakeup_reason; 4639 const char *wake_str; 4640 4641 /* if no wake event, nothing to print */ 4642 if (!wus) 4643 return; 4644 4645 if (wus & PFPM_WUS_LNKC_M) 4646 wake_str = "Link\n"; 4647 else if (wus & PFPM_WUS_MAG_M) 4648 wake_str = "Magic Packet\n"; 4649 else if (wus & PFPM_WUS_MNG_M) 4650 wake_str = "Management\n"; 4651 else if (wus & PFPM_WUS_FW_RST_WK_M) 4652 wake_str = "Firmware Reset\n"; 4653 else 4654 wake_str = "Unknown\n"; 4655 4656 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 4657} 4658 4659/** 4660 * ice_register_netdev - register netdev 4661 * @vsi: pointer to the VSI struct 4662 */ 4663static int ice_register_netdev(struct ice_vsi *vsi) 4664{ 4665 int err; 4666 4667 if (!vsi || !vsi->netdev) 4668 return -EIO; 4669 4670 err = register_netdev(vsi->netdev); 4671 if (err) 4672 return err; 4673 4674 set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4675 netif_carrier_off(vsi->netdev); 4676 netif_tx_stop_all_queues(vsi->netdev); 4677 4678 return 0; 4679} 4680 4681static void ice_unregister_netdev(struct ice_vsi *vsi) 4682{ 4683 if (!vsi || !vsi->netdev) 4684 return; 4685 4686 unregister_netdev(vsi->netdev); 4687 clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4688} 4689 4690/** 4691 * ice_cfg_netdev - Allocate, configure and register a netdev 4692 * @vsi: the VSI associated with the new netdev 4693 * 4694 * Returns 0 on success, negative value on failure 4695 */ 4696static int ice_cfg_netdev(struct ice_vsi *vsi) 4697{ 4698 struct ice_netdev_priv *np; 4699 struct net_device *netdev; 4700 u8 mac_addr[ETH_ALEN]; 4701 4702 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 4703 vsi->alloc_rxq); 4704 if (!netdev) 4705 return -ENOMEM; 4706 4707 set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 4708 vsi->netdev = netdev; 4709 np = netdev_priv(netdev); 4710 np->vsi = vsi; 4711 4712 ice_set_netdev_features(netdev); 4713 ice_set_ops(vsi); 4714 4715 if (vsi->type == ICE_VSI_PF) { 4716 SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back)); 4717 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 4718 eth_hw_addr_set(netdev, mac_addr); 4719 } 4720 4721 netdev->priv_flags |= IFF_UNICAST_FLT; 4722 4723 /* Setup netdev TC information */ 4724 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 4725 4726 netdev->max_mtu = ICE_MAX_MTU; 4727 4728 return 0; 4729} 4730 4731static void ice_decfg_netdev(struct ice_vsi *vsi) 4732{ 4733 clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 4734 free_netdev(vsi->netdev); 4735 vsi->netdev = NULL; 4736} 4737 4738void ice_init_dev_hw(struct ice_pf *pf) 4739{ 4740 struct ice_hw *hw = &pf->hw; 4741 int err; 4742 4743 ice_init_feature_support(pf); 4744 4745 err = ice_init_ddp_config(hw, pf); 4746 4747 /* if ice_init_ddp_config fails, ICE_FLAG_ADV_FEATURES bit won't be 4748 * set in pf->state, which will cause ice_is_safe_mode to return 4749 * true 4750 */ 4751 if (err || ice_is_safe_mode(pf)) { 4752 /* we already got function/device capabilities but these don't 4753 * reflect what the driver needs to do in safe mode. Instead of 4754 * adding conditional logic everywhere to ignore these 4755 * device/function capabilities, override them. 4756 */ 4757 ice_set_safe_mode_caps(hw); 4758 } 4759} 4760 4761int ice_init_dev(struct ice_pf *pf) 4762{ 4763 struct device *dev = ice_pf_to_dev(pf); 4764 int err; 4765 4766 ice_set_pf_caps(pf); 4767 err = ice_init_interrupt_scheme(pf); 4768 if (err) { 4769 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 4770 return -EIO; 4771 } 4772 4773 ice_start_service_task(pf); 4774 4775 return 0; 4776} 4777 4778void ice_deinit_dev(struct ice_pf *pf) 4779{ 4780 ice_service_task_stop(pf); 4781 4782 /* Service task is already stopped, so call reset directly. */ 4783 ice_reset(&pf->hw, ICE_RESET_PFR); 4784 pci_wait_for_pending_transaction(pf->pdev); 4785 ice_clear_interrupt_scheme(pf); 4786} 4787 4788static void ice_init_features(struct ice_pf *pf) 4789{ 4790 struct device *dev = ice_pf_to_dev(pf); 4791 4792 if (ice_is_safe_mode(pf)) 4793 return; 4794 4795 /* initialize DDP driven features */ 4796 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 4797 ice_ptp_init(pf); 4798 4799 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 4800 ice_gnss_init(pf); 4801 4802 if (ice_is_feature_supported(pf, ICE_F_CGU) || 4803 ice_is_feature_supported(pf, ICE_F_PHY_RCLK)) 4804 ice_dpll_init(pf); 4805 4806 /* Note: Flow director init failure is non-fatal to load */ 4807 if (ice_init_fdir(pf)) 4808 dev_err(dev, "could not initialize flow director\n"); 4809 4810 /* Note: DCB init failure is non-fatal to load */ 4811 if (ice_init_pf_dcb(pf, false)) { 4812 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 4813 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 4814 } else { 4815 ice_cfg_lldp_mib_change(&pf->hw, true); 4816 } 4817 4818 if (ice_init_lag(pf)) 4819 dev_warn(dev, "Failed to init link aggregation support\n"); 4820 4821 ice_hwmon_init(pf); 4822} 4823 4824static void ice_deinit_features(struct ice_pf *pf) 4825{ 4826 if (ice_is_safe_mode(pf)) 4827 return; 4828 4829 ice_deinit_lag(pf); 4830 if (test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags)) 4831 ice_cfg_lldp_mib_change(&pf->hw, false); 4832 ice_deinit_fdir(pf); 4833 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 4834 ice_gnss_exit(pf); 4835 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 4836 ice_ptp_release(pf); 4837 if (test_bit(ICE_FLAG_DPLL, pf->flags)) 4838 ice_dpll_deinit(pf); 4839 if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) 4840 xa_destroy(&pf->eswitch.reprs); 4841 ice_hwmon_exit(pf); 4842} 4843 4844static void ice_init_wakeup(struct ice_pf *pf) 4845{ 4846 /* Save wakeup reason register for later use */ 4847 pf->wakeup_reason = rd32(&pf->hw, PFPM_WUS); 4848 4849 /* check for a power management event */ 4850 ice_print_wake_reason(pf); 4851 4852 /* clear wake status, all bits */ 4853 wr32(&pf->hw, PFPM_WUS, U32_MAX); 4854 4855 /* Disable WoL at init, wait for user to enable */ 4856 device_set_wakeup_enable(ice_pf_to_dev(pf), false); 4857} 4858 4859static int ice_init_link(struct ice_pf *pf) 4860{ 4861 struct device *dev = ice_pf_to_dev(pf); 4862 int err; 4863 4864 err = ice_init_link_events(pf->hw.port_info); 4865 if (err) { 4866 dev_err(dev, "ice_init_link_events failed: %d\n", err); 4867 return err; 4868 } 4869 4870 /* not a fatal error if this fails */ 4871 err = ice_init_nvm_phy_type(pf->hw.port_info); 4872 if (err) 4873 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 4874 4875 /* not a fatal error if this fails */ 4876 err = ice_update_link_info(pf->hw.port_info); 4877 if (err) 4878 dev_err(dev, "ice_update_link_info failed: %d\n", err); 4879 4880 ice_init_link_dflt_override(pf->hw.port_info); 4881 4882 ice_check_link_cfg_err(pf, 4883 pf->hw.port_info->phy.link_info.link_cfg_err); 4884 4885 /* if media available, initialize PHY settings */ 4886 if (pf->hw.port_info->phy.link_info.link_info & 4887 ICE_AQ_MEDIA_AVAILABLE) { 4888 /* not a fatal error if this fails */ 4889 err = ice_init_phy_user_cfg(pf->hw.port_info); 4890 if (err) 4891 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 4892 4893 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 4894 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4895 4896 if (vsi) 4897 ice_configure_phy(vsi); 4898 } 4899 } else { 4900 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 4901 } 4902 4903 return err; 4904} 4905 4906static int ice_init_pf_sw(struct ice_pf *pf) 4907{ 4908 bool dvm = ice_is_dvm_ena(&pf->hw); 4909 struct ice_vsi *vsi; 4910 int err; 4911 4912 /* create switch struct for the switch element created by FW on boot */ 4913 pf->first_sw = kzalloc(sizeof(*pf->first_sw), GFP_KERNEL); 4914 if (!pf->first_sw) 4915 return -ENOMEM; 4916 4917 if (pf->hw.evb_veb) 4918 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 4919 else 4920 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 4921 4922 pf->first_sw->pf = pf; 4923 4924 /* record the sw_id available for later use */ 4925 pf->first_sw->sw_id = pf->hw.port_info->sw_id; 4926 4927 err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 4928 if (err) 4929 goto err_aq_set_port_params; 4930 4931 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 4932 if (!vsi) { 4933 err = -ENOMEM; 4934 goto err_pf_vsi_setup; 4935 } 4936 4937 return 0; 4938 4939err_pf_vsi_setup: 4940err_aq_set_port_params: 4941 kfree(pf->first_sw); 4942 return err; 4943} 4944 4945static void ice_deinit_pf_sw(struct ice_pf *pf) 4946{ 4947 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4948 4949 if (!vsi) 4950 return; 4951 4952 ice_vsi_release(vsi); 4953 kfree(pf->first_sw); 4954} 4955 4956static int ice_alloc_vsis(struct ice_pf *pf) 4957{ 4958 struct device *dev = ice_pf_to_dev(pf); 4959 4960 pf->num_alloc_vsi = pf->hw.func_caps.guar_num_vsi; 4961 if (!pf->num_alloc_vsi) 4962 return -EIO; 4963 4964 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 4965 dev_warn(dev, 4966 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 4967 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 4968 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 4969 } 4970 4971 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 4972 GFP_KERNEL); 4973 if (!pf->vsi) 4974 return -ENOMEM; 4975 4976 pf->vsi_stats = devm_kcalloc(dev, pf->num_alloc_vsi, 4977 sizeof(*pf->vsi_stats), GFP_KERNEL); 4978 if (!pf->vsi_stats) { 4979 devm_kfree(dev, pf->vsi); 4980 return -ENOMEM; 4981 } 4982 4983 return 0; 4984} 4985 4986static void ice_dealloc_vsis(struct ice_pf *pf) 4987{ 4988 devm_kfree(ice_pf_to_dev(pf), pf->vsi_stats); 4989 pf->vsi_stats = NULL; 4990 4991 pf->num_alloc_vsi = 0; 4992 devm_kfree(ice_pf_to_dev(pf), pf->vsi); 4993 pf->vsi = NULL; 4994} 4995 4996static int ice_init_devlink(struct ice_pf *pf) 4997{ 4998 int err; 4999 5000 err = ice_devlink_register_params(pf); 5001 if (err) 5002 return err; 5003 5004 ice_devlink_init_regions(pf); 5005 ice_devlink_register(pf); 5006 ice_health_init(pf); 5007 5008 return 0; 5009} 5010 5011static void ice_deinit_devlink(struct ice_pf *pf) 5012{ 5013 ice_health_deinit(pf); 5014 ice_devlink_unregister(pf); 5015 ice_devlink_destroy_regions(pf); 5016 ice_devlink_unregister_params(pf); 5017} 5018 5019static int ice_init(struct ice_pf *pf) 5020{ 5021 struct device *dev = ice_pf_to_dev(pf); 5022 int err; 5023 5024 err = ice_init_pf(pf); 5025 if (err) { 5026 dev_err(dev, "ice_init_pf failed: %d\n", err); 5027 return err; 5028 } 5029 5030 if (pf->hw.mac_type == ICE_MAC_E830) { 5031 err = pci_enable_ptm(pf->pdev, NULL); 5032 if (err) 5033 dev_dbg(dev, "PCIe PTM not supported by PCIe bus/controller\n"); 5034 } 5035 5036 err = ice_alloc_vsis(pf); 5037 if (err) 5038 goto unroll_pf_init; 5039 5040 err = ice_init_pf_sw(pf); 5041 if (err) 5042 goto err_init_pf_sw; 5043 5044 ice_init_wakeup(pf); 5045 5046 err = ice_init_link(pf); 5047 if (err) 5048 goto err_init_link; 5049 5050 err = ice_send_version(pf); 5051 if (err) 5052 goto err_init_link; 5053 5054 ice_verify_cacheline_size(pf); 5055 5056 if (ice_is_safe_mode(pf)) 5057 ice_set_safe_mode_vlan_cfg(pf); 5058 else 5059 /* print PCI link speed and width */ 5060 pcie_print_link_status(pf->pdev); 5061 5062 /* ready to go, so clear down state bit */ 5063 clear_bit(ICE_DOWN, pf->state); 5064 clear_bit(ICE_SERVICE_DIS, pf->state); 5065 5066 /* since everything is good, start the service timer */ 5067 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5068 5069 return 0; 5070 5071err_init_link: 5072 ice_deinit_pf_sw(pf); 5073err_init_pf_sw: 5074 ice_dealloc_vsis(pf); 5075unroll_pf_init: 5076 ice_deinit_pf(pf); 5077 return err; 5078} 5079 5080static void ice_deinit(struct ice_pf *pf) 5081{ 5082 set_bit(ICE_SERVICE_DIS, pf->state); 5083 set_bit(ICE_DOWN, pf->state); 5084 5085 ice_deinit_pf_sw(pf); 5086 ice_dealloc_vsis(pf); 5087 ice_deinit_pf(pf); 5088} 5089 5090/** 5091 * ice_load - load pf by init hw and starting VSI 5092 * @pf: pointer to the pf instance 5093 * 5094 * This function has to be called under devl_lock. 5095 */ 5096int ice_load(struct ice_pf *pf) 5097{ 5098 struct ice_vsi *vsi; 5099 int err; 5100 5101 devl_assert_locked(priv_to_devlink(pf)); 5102 5103 vsi = ice_get_main_vsi(pf); 5104 5105 /* init channel list */ 5106 INIT_LIST_HEAD(&vsi->ch_list); 5107 5108 err = ice_cfg_netdev(vsi); 5109 if (err) 5110 return err; 5111 5112 /* Setup DCB netlink interface */ 5113 ice_dcbnl_setup(vsi); 5114 5115 err = ice_init_mac_fltr(pf); 5116 if (err) 5117 goto err_init_mac_fltr; 5118 5119 err = ice_devlink_create_pf_port(pf); 5120 if (err) 5121 goto err_devlink_create_pf_port; 5122 5123 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port); 5124 5125 err = ice_register_netdev(vsi); 5126 if (err) 5127 goto err_register_netdev; 5128 5129 err = ice_tc_indir_block_register(vsi); 5130 if (err) 5131 goto err_tc_indir_block_register; 5132 5133 ice_napi_add(vsi); 5134 5135 ice_init_features(pf); 5136 5137 err = ice_init_rdma(pf); 5138 if (err) 5139 goto err_init_rdma; 5140 5141 /* Finalize RDMA: VSI already created, assign info and plug device */ 5142 ice_rdma_finalize_setup(pf); 5143 5144 ice_service_task_restart(pf); 5145 5146 clear_bit(ICE_DOWN, pf->state); 5147 5148 return 0; 5149 5150err_init_rdma: 5151 ice_deinit_features(pf); 5152 ice_tc_indir_block_unregister(vsi); 5153err_tc_indir_block_register: 5154 ice_unregister_netdev(vsi); 5155err_register_netdev: 5156 ice_devlink_destroy_pf_port(pf); 5157err_devlink_create_pf_port: 5158err_init_mac_fltr: 5159 ice_decfg_netdev(vsi); 5160 return err; 5161} 5162 5163/** 5164 * ice_unload - unload pf by stopping VSI and deinit hw 5165 * @pf: pointer to the pf instance 5166 * 5167 * This function has to be called under devl_lock. 5168 */ 5169void ice_unload(struct ice_pf *pf) 5170{ 5171 struct ice_vsi *vsi = ice_get_main_vsi(pf); 5172 5173 devl_assert_locked(priv_to_devlink(pf)); 5174 5175 ice_unplug_aux_dev(pf); 5176 ice_deinit_rdma(pf); 5177 ice_deinit_features(pf); 5178 ice_tc_indir_block_unregister(vsi); 5179 ice_unregister_netdev(vsi); 5180 ice_devlink_destroy_pf_port(pf); 5181 ice_decfg_netdev(vsi); 5182} 5183 5184static int ice_probe_recovery_mode(struct ice_pf *pf) 5185{ 5186 struct device *dev = ice_pf_to_dev(pf); 5187 int err; 5188 5189 dev_err(dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode\n"); 5190 5191 INIT_HLIST_HEAD(&pf->aq_wait_list); 5192 spin_lock_init(&pf->aq_wait_lock); 5193 init_waitqueue_head(&pf->aq_wait_queue); 5194 5195 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 5196 pf->serv_tmr_period = HZ; 5197 INIT_WORK(&pf->serv_task, ice_service_task_recovery_mode); 5198 clear_bit(ICE_SERVICE_SCHED, pf->state); 5199 err = ice_create_all_ctrlq(&pf->hw); 5200 if (err) 5201 return err; 5202 5203 scoped_guard(devl, priv_to_devlink(pf)) { 5204 err = ice_init_devlink(pf); 5205 if (err) 5206 return err; 5207 } 5208 5209 ice_service_task_restart(pf); 5210 5211 return 0; 5212} 5213 5214/** 5215 * ice_probe - Device initialization routine 5216 * @pdev: PCI device information struct 5217 * @ent: entry in ice_pci_tbl 5218 * 5219 * Returns 0 on success, negative on failure 5220 */ 5221static int 5222ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 5223{ 5224 struct device *dev = &pdev->dev; 5225 bool need_dev_deinit = false; 5226 struct ice_adapter *adapter; 5227 struct ice_pf *pf; 5228 struct ice_hw *hw; 5229 int err; 5230 5231 if (pdev->is_virtfn) { 5232 dev_err(dev, "can't probe a virtual function\n"); 5233 return -EINVAL; 5234 } 5235 5236 /* when under a kdump kernel initiate a reset before enabling the 5237 * device in order to clear out any pending DMA transactions. These 5238 * transactions can cause some systems to machine check when doing 5239 * the pcim_enable_device() below. 5240 */ 5241 if (is_kdump_kernel()) { 5242 pci_save_state(pdev); 5243 pci_clear_master(pdev); 5244 err = pcie_flr(pdev); 5245 if (err) 5246 return err; 5247 pci_restore_state(pdev); 5248 } 5249 5250 /* this driver uses devres, see 5251 * Documentation/driver-api/driver-model/devres.rst 5252 */ 5253 err = pcim_enable_device(pdev); 5254 if (err) 5255 return err; 5256 5257 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev)); 5258 if (err) { 5259 dev_err(dev, "BAR0 I/O map error %d\n", err); 5260 return err; 5261 } 5262 5263 pf = ice_allocate_pf(dev); 5264 if (!pf) 5265 return -ENOMEM; 5266 5267 /* initialize Auxiliary index to invalid value */ 5268 pf->aux_idx = -1; 5269 5270 /* set up for high or low DMA */ 5271 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 5272 if (err) { 5273 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 5274 return err; 5275 } 5276 5277 pci_set_master(pdev); 5278 pf->pdev = pdev; 5279 pci_set_drvdata(pdev, pf); 5280 set_bit(ICE_DOWN, pf->state); 5281 /* Disable service task until DOWN bit is cleared */ 5282 set_bit(ICE_SERVICE_DIS, pf->state); 5283 5284 hw = &pf->hw; 5285 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 5286 pci_save_state(pdev); 5287 5288 hw->back = pf; 5289 hw->port_info = NULL; 5290 hw->vendor_id = pdev->vendor; 5291 hw->device_id = pdev->device; 5292 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 5293 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5294 hw->subsystem_device_id = pdev->subsystem_device; 5295 hw->bus.device = PCI_SLOT(pdev->devfn); 5296 hw->bus.func = PCI_FUNC(pdev->devfn); 5297 ice_set_ctrlq_len(hw); 5298 5299 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 5300 5301#ifndef CONFIG_DYNAMIC_DEBUG 5302 if (debug < -1) 5303 hw->debug_mask = debug; 5304#endif 5305 5306 if (ice_is_recovery_mode(hw)) 5307 return ice_probe_recovery_mode(pf); 5308 5309 err = ice_init_hw(hw); 5310 if (err) { 5311 dev_err(dev, "ice_init_hw failed: %d\n", err); 5312 return err; 5313 } 5314 5315 adapter = ice_adapter_get(pdev); 5316 if (IS_ERR(adapter)) { 5317 err = PTR_ERR(adapter); 5318 goto unroll_hw_init; 5319 } 5320 pf->adapter = adapter; 5321 5322 err = ice_init_dev(pf); 5323 if (err) 5324 goto unroll_adapter; 5325 5326 err = ice_init(pf); 5327 if (err) 5328 goto unroll_dev_init; 5329 5330 devl_lock(priv_to_devlink(pf)); 5331 err = ice_load(pf); 5332 if (err) 5333 goto unroll_init; 5334 5335 err = ice_init_devlink(pf); 5336 if (err) 5337 goto unroll_load; 5338 devl_unlock(priv_to_devlink(pf)); 5339 5340 return 0; 5341 5342unroll_load: 5343 ice_unload(pf); 5344unroll_init: 5345 devl_unlock(priv_to_devlink(pf)); 5346 ice_deinit(pf); 5347unroll_dev_init: 5348 need_dev_deinit = true; 5349unroll_adapter: 5350 ice_adapter_put(pdev); 5351unroll_hw_init: 5352 ice_deinit_hw(hw); 5353 if (need_dev_deinit) 5354 ice_deinit_dev(pf); 5355 return err; 5356} 5357 5358/** 5359 * ice_set_wake - enable or disable Wake on LAN 5360 * @pf: pointer to the PF struct 5361 * 5362 * Simple helper for WoL control 5363 */ 5364static void ice_set_wake(struct ice_pf *pf) 5365{ 5366 struct ice_hw *hw = &pf->hw; 5367 bool wol = pf->wol_ena; 5368 5369 /* clear wake state, otherwise new wake events won't fire */ 5370 wr32(hw, PFPM_WUS, U32_MAX); 5371 5372 /* enable / disable APM wake up, no RMW needed */ 5373 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 5374 5375 /* set magic packet filter enabled */ 5376 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 5377} 5378 5379/** 5380 * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet 5381 * @pf: pointer to the PF struct 5382 * 5383 * Issue firmware command to enable multicast magic wake, making 5384 * sure that any locally administered address (LAA) is used for 5385 * wake, and that PF reset doesn't undo the LAA. 5386 */ 5387static void ice_setup_mc_magic_wake(struct ice_pf *pf) 5388{ 5389 struct device *dev = ice_pf_to_dev(pf); 5390 struct ice_hw *hw = &pf->hw; 5391 u8 mac_addr[ETH_ALEN]; 5392 struct ice_vsi *vsi; 5393 int status; 5394 u8 flags; 5395 5396 if (!pf->wol_ena) 5397 return; 5398 5399 vsi = ice_get_main_vsi(pf); 5400 if (!vsi) 5401 return; 5402 5403 /* Get current MAC address in case it's an LAA */ 5404 if (vsi->netdev) 5405 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 5406 else 5407 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 5408 5409 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 5410 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 5411 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 5412 5413 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 5414 if (status) 5415 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n", 5416 status, libie_aq_str(hw->adminq.sq_last_status)); 5417} 5418 5419/** 5420 * ice_remove - Device removal routine 5421 * @pdev: PCI device information struct 5422 */ 5423static void ice_remove(struct pci_dev *pdev) 5424{ 5425 struct ice_pf *pf = pci_get_drvdata(pdev); 5426 int i; 5427 5428 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 5429 if (!ice_is_reset_in_progress(pf->state)) 5430 break; 5431 msleep(100); 5432 } 5433 5434 if (ice_is_recovery_mode(&pf->hw)) { 5435 ice_service_task_stop(pf); 5436 scoped_guard(devl, priv_to_devlink(pf)) { 5437 ice_deinit_devlink(pf); 5438 } 5439 return; 5440 } 5441 5442 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 5443 set_bit(ICE_VF_RESETS_DISABLED, pf->state); 5444 ice_free_vfs(pf); 5445 } 5446 5447 if (!ice_is_safe_mode(pf)) 5448 ice_remove_arfs(pf); 5449 5450 devl_lock(priv_to_devlink(pf)); 5451 ice_dealloc_all_dynamic_ports(pf); 5452 ice_deinit_devlink(pf); 5453 5454 ice_unload(pf); 5455 devl_unlock(priv_to_devlink(pf)); 5456 5457 ice_deinit(pf); 5458 ice_vsi_release_all(pf); 5459 5460 ice_setup_mc_magic_wake(pf); 5461 ice_set_wake(pf); 5462 5463 ice_adapter_put(pdev); 5464 ice_deinit_hw(&pf->hw); 5465 5466 ice_deinit_dev(pf); 5467 ice_aq_cancel_waiting_tasks(pf); 5468 set_bit(ICE_DOWN, pf->state); 5469} 5470 5471/** 5472 * ice_shutdown - PCI callback for shutting down device 5473 * @pdev: PCI device information struct 5474 */ 5475static void ice_shutdown(struct pci_dev *pdev) 5476{ 5477 struct ice_pf *pf = pci_get_drvdata(pdev); 5478 5479 ice_remove(pdev); 5480 5481 if (system_state == SYSTEM_POWER_OFF) { 5482 pci_wake_from_d3(pdev, pf->wol_ena); 5483 pci_set_power_state(pdev, PCI_D3hot); 5484 } 5485} 5486 5487/** 5488 * ice_prepare_for_shutdown - prep for PCI shutdown 5489 * @pf: board private structure 5490 * 5491 * Inform or close all dependent features in prep for PCI device shutdown 5492 */ 5493static void ice_prepare_for_shutdown(struct ice_pf *pf) 5494{ 5495 struct ice_hw *hw = &pf->hw; 5496 u32 v; 5497 5498 /* Notify VFs of impending reset */ 5499 if (ice_check_sq_alive(hw, &hw->mailboxq)) 5500 ice_vc_notify_reset(pf); 5501 5502 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 5503 5504 /* disable the VSIs and their queues that are not already DOWN */ 5505 ice_pf_dis_all_vsi(pf, false); 5506 5507 ice_for_each_vsi(pf, v) 5508 if (pf->vsi[v]) 5509 pf->vsi[v]->vsi_num = 0; 5510 5511 ice_shutdown_all_ctrlq(hw, true); 5512} 5513 5514/** 5515 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 5516 * @pf: board private structure to reinitialize 5517 * 5518 * This routine reinitialize interrupt scheme that was cleared during 5519 * power management suspend callback. 5520 * 5521 * This should be called during resume routine to re-allocate the q_vectors 5522 * and reacquire interrupts. 5523 */ 5524static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 5525{ 5526 struct device *dev = ice_pf_to_dev(pf); 5527 int ret, v; 5528 5529 /* Since we clear MSIX flag during suspend, we need to 5530 * set it back during resume... 5531 */ 5532 5533 ret = ice_init_interrupt_scheme(pf); 5534 if (ret) { 5535 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 5536 return ret; 5537 } 5538 5539 /* Remap vectors and rings, after successful re-init interrupts */ 5540 ice_for_each_vsi(pf, v) { 5541 if (!pf->vsi[v]) 5542 continue; 5543 5544 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 5545 if (ret) 5546 goto err_reinit; 5547 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 5548 rtnl_lock(); 5549 ice_vsi_set_napi_queues(pf->vsi[v]); 5550 rtnl_unlock(); 5551 } 5552 5553 ret = ice_req_irq_msix_misc(pf); 5554 if (ret) { 5555 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 5556 ret); 5557 goto err_reinit; 5558 } 5559 5560 return 0; 5561 5562err_reinit: 5563 while (v--) 5564 if (pf->vsi[v]) { 5565 rtnl_lock(); 5566 ice_vsi_clear_napi_queues(pf->vsi[v]); 5567 rtnl_unlock(); 5568 ice_vsi_free_q_vectors(pf->vsi[v]); 5569 } 5570 5571 return ret; 5572} 5573 5574/** 5575 * ice_suspend 5576 * @dev: generic device information structure 5577 * 5578 * Power Management callback to quiesce the device and prepare 5579 * for D3 transition. 5580 */ 5581static int ice_suspend(struct device *dev) 5582{ 5583 struct pci_dev *pdev = to_pci_dev(dev); 5584 struct ice_pf *pf; 5585 int disabled, v; 5586 5587 pf = pci_get_drvdata(pdev); 5588 5589 if (!ice_pf_state_is_nominal(pf)) { 5590 dev_err(dev, "Device is not ready, no need to suspend it\n"); 5591 return -EBUSY; 5592 } 5593 5594 /* Stop watchdog tasks until resume completion. 5595 * Even though it is most likely that the service task is 5596 * disabled if the device is suspended or down, the service task's 5597 * state is controlled by a different state bit, and we should 5598 * store and honor whatever state that bit is in at this point. 5599 */ 5600 disabled = ice_service_task_stop(pf); 5601 5602 ice_unplug_aux_dev(pf); 5603 ice_deinit_rdma(pf); 5604 5605 /* Already suspended?, then there is nothing to do */ 5606 if (test_and_set_bit(ICE_SUSPENDED, pf->state)) { 5607 if (!disabled) 5608 ice_service_task_restart(pf); 5609 return 0; 5610 } 5611 5612 if (test_bit(ICE_DOWN, pf->state) || 5613 ice_is_reset_in_progress(pf->state)) { 5614 dev_err(dev, "can't suspend device in reset or already down\n"); 5615 if (!disabled) 5616 ice_service_task_restart(pf); 5617 return 0; 5618 } 5619 5620 ice_setup_mc_magic_wake(pf); 5621 5622 ice_prepare_for_shutdown(pf); 5623 5624 ice_set_wake(pf); 5625 5626 /* Free vectors, clear the interrupt scheme and release IRQs 5627 * for proper hibernation, especially with large number of CPUs. 5628 * Otherwise hibernation might fail when mapping all the vectors back 5629 * to CPU0. 5630 */ 5631 ice_free_irq_msix_misc(pf); 5632 ice_for_each_vsi(pf, v) { 5633 if (!pf->vsi[v]) 5634 continue; 5635 rtnl_lock(); 5636 ice_vsi_clear_napi_queues(pf->vsi[v]); 5637 rtnl_unlock(); 5638 ice_vsi_free_q_vectors(pf->vsi[v]); 5639 } 5640 ice_clear_interrupt_scheme(pf); 5641 5642 pci_save_state(pdev); 5643 pci_wake_from_d3(pdev, pf->wol_ena); 5644 pci_set_power_state(pdev, PCI_D3hot); 5645 return 0; 5646} 5647 5648/** 5649 * ice_resume - PM callback for waking up from D3 5650 * @dev: generic device information structure 5651 */ 5652static int ice_resume(struct device *dev) 5653{ 5654 struct pci_dev *pdev = to_pci_dev(dev); 5655 enum ice_reset_req reset_type; 5656 struct ice_pf *pf; 5657 struct ice_hw *hw; 5658 int ret; 5659 5660 pci_set_power_state(pdev, PCI_D0); 5661 pci_restore_state(pdev); 5662 5663 if (!pci_device_is_present(pdev)) 5664 return -ENODEV; 5665 5666 ret = pci_enable_device_mem(pdev); 5667 if (ret) { 5668 dev_err(dev, "Cannot enable device after suspend\n"); 5669 return ret; 5670 } 5671 5672 pf = pci_get_drvdata(pdev); 5673 hw = &pf->hw; 5674 5675 pf->wakeup_reason = rd32(hw, PFPM_WUS); 5676 ice_print_wake_reason(pf); 5677 5678 /* We cleared the interrupt scheme when we suspended, so we need to 5679 * restore it now to resume device functionality. 5680 */ 5681 ret = ice_reinit_interrupt_scheme(pf); 5682 if (ret) 5683 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 5684 5685 ret = ice_init_rdma(pf); 5686 if (ret) 5687 dev_err(dev, "Reinitialize RDMA during resume failed: %d\n", 5688 ret); 5689 5690 clear_bit(ICE_DOWN, pf->state); 5691 /* Now perform PF reset and rebuild */ 5692 reset_type = ICE_RESET_PFR; 5693 /* re-enable service task for reset, but allow reset to schedule it */ 5694 clear_bit(ICE_SERVICE_DIS, pf->state); 5695 5696 if (ice_schedule_reset(pf, reset_type)) 5697 dev_err(dev, "Reset during resume failed.\n"); 5698 5699 clear_bit(ICE_SUSPENDED, pf->state); 5700 ice_service_task_restart(pf); 5701 5702 /* Restart the service task */ 5703 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5704 5705 return 0; 5706} 5707 5708/** 5709 * ice_pci_err_detected - warning that PCI error has been detected 5710 * @pdev: PCI device information struct 5711 * @err: the type of PCI error 5712 * 5713 * Called to warn that something happened on the PCI bus and the error handling 5714 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 5715 */ 5716static pci_ers_result_t 5717ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err) 5718{ 5719 struct ice_pf *pf = pci_get_drvdata(pdev); 5720 5721 if (!pf) { 5722 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 5723 __func__, err); 5724 return PCI_ERS_RESULT_DISCONNECT; 5725 } 5726 5727 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5728 ice_service_task_stop(pf); 5729 5730 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5731 set_bit(ICE_PFR_REQ, pf->state); 5732 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5733 } 5734 } 5735 5736 return PCI_ERS_RESULT_NEED_RESET; 5737} 5738 5739/** 5740 * ice_pci_err_slot_reset - a PCI slot reset has just happened 5741 * @pdev: PCI device information struct 5742 * 5743 * Called to determine if the driver can recover from the PCI slot reset by 5744 * using a register read to determine if the device is recoverable. 5745 */ 5746static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 5747{ 5748 struct ice_pf *pf = pci_get_drvdata(pdev); 5749 pci_ers_result_t result; 5750 int err; 5751 u32 reg; 5752 5753 err = pci_enable_device_mem(pdev); 5754 if (err) { 5755 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 5756 err); 5757 result = PCI_ERS_RESULT_DISCONNECT; 5758 } else { 5759 pci_set_master(pdev); 5760 pci_restore_state(pdev); 5761 pci_wake_from_d3(pdev, false); 5762 5763 /* Check for life */ 5764 reg = rd32(&pf->hw, GLGEN_RTRIG); 5765 if (!reg) 5766 result = PCI_ERS_RESULT_RECOVERED; 5767 else 5768 result = PCI_ERS_RESULT_DISCONNECT; 5769 } 5770 5771 return result; 5772} 5773 5774/** 5775 * ice_pci_err_resume - restart operations after PCI error recovery 5776 * @pdev: PCI device information struct 5777 * 5778 * Called to allow the driver to bring things back up after PCI error and/or 5779 * reset recovery have finished 5780 */ 5781static void ice_pci_err_resume(struct pci_dev *pdev) 5782{ 5783 struct ice_pf *pf = pci_get_drvdata(pdev); 5784 5785 if (!pf) { 5786 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 5787 __func__); 5788 return; 5789 } 5790 5791 if (test_bit(ICE_SUSPENDED, pf->state)) { 5792 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 5793 __func__); 5794 return; 5795 } 5796 5797 ice_restore_all_vfs_msi_state(pf); 5798 5799 ice_do_reset(pf, ICE_RESET_PFR); 5800 ice_service_task_restart(pf); 5801 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5802} 5803 5804/** 5805 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 5806 * @pdev: PCI device information struct 5807 */ 5808static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 5809{ 5810 struct ice_pf *pf = pci_get_drvdata(pdev); 5811 5812 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5813 ice_service_task_stop(pf); 5814 5815 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5816 set_bit(ICE_PFR_REQ, pf->state); 5817 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5818 } 5819 } 5820} 5821 5822/** 5823 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 5824 * @pdev: PCI device information struct 5825 */ 5826static void ice_pci_err_reset_done(struct pci_dev *pdev) 5827{ 5828 ice_pci_err_resume(pdev); 5829} 5830 5831/* ice_pci_tbl - PCI Device ID Table 5832 * 5833 * Wildcard entries (PCI_ANY_ID) should come last 5834 * Last entry must be all 0s 5835 * 5836 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 5837 * Class, Class Mask, private data (not used) } 5838 */ 5839static const struct pci_device_id ice_pci_tbl[] = { 5840 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE) }, 5841 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP) }, 5842 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP) }, 5843 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE) }, 5844 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP) }, 5845 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP) }, 5846 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE) }, 5847 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP) }, 5848 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP) }, 5849 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T) }, 5850 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII) }, 5851 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE) }, 5852 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP) }, 5853 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP) }, 5854 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T) }, 5855 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII) }, 5856 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE) }, 5857 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP) }, 5858 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T) }, 5859 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII) }, 5860 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE) }, 5861 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP) }, 5862 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T) }, 5863 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE) }, 5864 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP) }, 5865 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT) }, 5866 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E825C_BACKPLANE), }, 5867 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E825C_QSFP), }, 5868 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E825C_SFP), }, 5869 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E825C_SGMII), }, 5870 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830CC_BACKPLANE) }, 5871 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830CC_QSFP56) }, 5872 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830CC_SFP) }, 5873 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830CC_SFP_DD) }, 5874 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830C_BACKPLANE), }, 5875 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_XXV_BACKPLANE), }, 5876 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830C_QSFP), }, 5877 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_XXV_QSFP), }, 5878 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830C_SFP), }, 5879 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E830_XXV_SFP), }, 5880 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835CC_BACKPLANE), }, 5881 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835CC_QSFP56), }, 5882 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835CC_SFP), }, 5883 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835C_BACKPLANE), }, 5884 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835C_QSFP), }, 5885 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835C_SFP), }, 5886 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835_L_BACKPLANE), }, 5887 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835_L_QSFP), }, 5888 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E835_L_SFP), }, 5889 /* required last entry */ 5890 {} 5891}; 5892MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 5893 5894static DEFINE_SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 5895 5896static const struct pci_error_handlers ice_pci_err_handler = { 5897 .error_detected = ice_pci_err_detected, 5898 .slot_reset = ice_pci_err_slot_reset, 5899 .reset_prepare = ice_pci_err_reset_prepare, 5900 .reset_done = ice_pci_err_reset_done, 5901 .resume = ice_pci_err_resume 5902}; 5903 5904static struct pci_driver ice_driver = { 5905 .name = KBUILD_MODNAME, 5906 .id_table = ice_pci_tbl, 5907 .probe = ice_probe, 5908 .remove = ice_remove, 5909 .driver.pm = pm_sleep_ptr(&ice_pm_ops), 5910 .shutdown = ice_shutdown, 5911 .sriov_configure = ice_sriov_configure, 5912 .sriov_get_vf_total_msix = ice_sriov_get_vf_total_msix, 5913 .sriov_set_msix_vec_count = ice_sriov_set_msix_vec_count, 5914 .err_handler = &ice_pci_err_handler 5915}; 5916 5917/** 5918 * ice_module_init - Driver registration routine 5919 * 5920 * ice_module_init is the first routine called when the driver is 5921 * loaded. All it does is register with the PCI subsystem. 5922 */ 5923static int __init ice_module_init(void) 5924{ 5925 int status = -ENOMEM; 5926 5927 pr_info("%s\n", ice_driver_string); 5928 pr_info("%s\n", ice_copyright); 5929 5930 ice_adv_lnk_speed_maps_init(); 5931 5932 ice_wq = alloc_workqueue("%s", WQ_UNBOUND, 0, KBUILD_MODNAME); 5933 if (!ice_wq) { 5934 pr_err("Failed to create workqueue\n"); 5935 return status; 5936 } 5937 5938 ice_lag_wq = alloc_ordered_workqueue("ice_lag_wq", 0); 5939 if (!ice_lag_wq) { 5940 pr_err("Failed to create LAG workqueue\n"); 5941 goto err_dest_wq; 5942 } 5943 5944 ice_debugfs_init(); 5945 5946 status = pci_register_driver(&ice_driver); 5947 if (status) { 5948 pr_err("failed to register PCI driver, err %d\n", status); 5949 goto err_dest_lag_wq; 5950 } 5951 5952 status = ice_sf_driver_register(); 5953 if (status) { 5954 pr_err("Failed to register SF driver, err %d\n", status); 5955 goto err_sf_driver; 5956 } 5957 5958 return 0; 5959 5960err_sf_driver: 5961 pci_unregister_driver(&ice_driver); 5962err_dest_lag_wq: 5963 destroy_workqueue(ice_lag_wq); 5964 ice_debugfs_exit(); 5965err_dest_wq: 5966 destroy_workqueue(ice_wq); 5967 return status; 5968} 5969module_init(ice_module_init); 5970 5971/** 5972 * ice_module_exit - Driver exit cleanup routine 5973 * 5974 * ice_module_exit is called just before the driver is removed 5975 * from memory. 5976 */ 5977static void __exit ice_module_exit(void) 5978{ 5979 ice_sf_driver_unregister(); 5980 pci_unregister_driver(&ice_driver); 5981 ice_debugfs_exit(); 5982 destroy_workqueue(ice_wq); 5983 destroy_workqueue(ice_lag_wq); 5984 pr_info("module unloaded\n"); 5985} 5986module_exit(ice_module_exit); 5987 5988/** 5989 * ice_set_mac_address - NDO callback to set MAC address 5990 * @netdev: network interface device structure 5991 * @pi: pointer to an address structure 5992 * 5993 * Returns 0 on success, negative on failure 5994 */ 5995static int ice_set_mac_address(struct net_device *netdev, void *pi) 5996{ 5997 struct ice_netdev_priv *np = netdev_priv(netdev); 5998 struct ice_vsi *vsi = np->vsi; 5999 struct ice_pf *pf = vsi->back; 6000 struct ice_hw *hw = &pf->hw; 6001 struct sockaddr *addr = pi; 6002 u8 old_mac[ETH_ALEN]; 6003 u8 flags = 0; 6004 u8 *mac; 6005 int err; 6006 6007 mac = (u8 *)addr->sa_data; 6008 6009 if (!is_valid_ether_addr(mac)) 6010 return -EADDRNOTAVAIL; 6011 6012 if (test_bit(ICE_DOWN, pf->state) || 6013 ice_is_reset_in_progress(pf->state)) { 6014 netdev_err(netdev, "can't set mac %pM. device not ready\n", 6015 mac); 6016 return -EBUSY; 6017 } 6018 6019 if (ice_chnl_dmac_fltr_cnt(pf)) { 6020 netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n", 6021 mac); 6022 return -EAGAIN; 6023 } 6024 6025 netif_addr_lock_bh(netdev); 6026 ether_addr_copy(old_mac, netdev->dev_addr); 6027 /* change the netdev's MAC address */ 6028 eth_hw_addr_set(netdev, mac); 6029 netif_addr_unlock_bh(netdev); 6030 6031 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 6032 err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI); 6033 if (err && err != -ENOENT) { 6034 err = -EADDRNOTAVAIL; 6035 goto err_update_filters; 6036 } 6037 6038 /* Add filter for new MAC. If filter exists, return success */ 6039 err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 6040 if (err == -EEXIST) { 6041 /* Although this MAC filter is already present in hardware it's 6042 * possible in some cases (e.g. bonding) that dev_addr was 6043 * modified outside of the driver and needs to be restored back 6044 * to this value. 6045 */ 6046 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 6047 6048 return 0; 6049 } else if (err) { 6050 /* error if the new filter addition failed */ 6051 err = -EADDRNOTAVAIL; 6052 } 6053 6054err_update_filters: 6055 if (err) { 6056 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 6057 mac); 6058 netif_addr_lock_bh(netdev); 6059 eth_hw_addr_set(netdev, old_mac); 6060 netif_addr_unlock_bh(netdev); 6061 return err; 6062 } 6063 6064 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 6065 netdev->dev_addr); 6066 6067 /* write new MAC address to the firmware */ 6068 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 6069 err = ice_aq_manage_mac_write(hw, mac, flags, NULL); 6070 if (err) { 6071 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n", 6072 mac, err); 6073 } 6074 return 0; 6075} 6076 6077/** 6078 * ice_set_rx_mode - NDO callback to set the netdev filters 6079 * @netdev: network interface device structure 6080 */ 6081static void ice_set_rx_mode(struct net_device *netdev) 6082{ 6083 struct ice_netdev_priv *np = netdev_priv(netdev); 6084 struct ice_vsi *vsi = np->vsi; 6085 6086 if (!vsi || ice_is_switchdev_running(vsi->back)) 6087 return; 6088 6089 /* Set the flags to synchronize filters 6090 * ndo_set_rx_mode may be triggered even without a change in netdev 6091 * flags 6092 */ 6093 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 6094 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 6095 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 6096 6097 /* schedule our worker thread which will take care of 6098 * applying the new filter changes 6099 */ 6100 ice_service_task_schedule(vsi->back); 6101} 6102 6103/** 6104 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 6105 * @netdev: network interface device structure 6106 * @queue_index: Queue ID 6107 * @maxrate: maximum bandwidth in Mbps 6108 */ 6109static int 6110ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 6111{ 6112 struct ice_netdev_priv *np = netdev_priv(netdev); 6113 struct ice_vsi *vsi = np->vsi; 6114 u16 q_handle; 6115 int status; 6116 u8 tc; 6117 6118 /* Validate maxrate requested is within permitted range */ 6119 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 6120 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 6121 maxrate, queue_index); 6122 return -EINVAL; 6123 } 6124 6125 q_handle = vsi->tx_rings[queue_index]->q_handle; 6126 tc = ice_dcb_get_tc(vsi, queue_index); 6127 6128 vsi = ice_locate_vsi_using_queue(vsi, queue_index); 6129 if (!vsi) { 6130 netdev_err(netdev, "Invalid VSI for given queue %d\n", 6131 queue_index); 6132 return -EINVAL; 6133 } 6134 6135 /* Set BW back to default, when user set maxrate to 0 */ 6136 if (!maxrate) 6137 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 6138 q_handle, ICE_MAX_BW); 6139 else 6140 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 6141 q_handle, ICE_MAX_BW, maxrate * 1000); 6142 if (status) 6143 netdev_err(netdev, "Unable to set Tx max rate, error %d\n", 6144 status); 6145 6146 return status; 6147} 6148 6149/** 6150 * ice_fdb_add - add an entry to the hardware database 6151 * @ndm: the input from the stack 6152 * @tb: pointer to array of nladdr (unused) 6153 * @dev: the net device pointer 6154 * @addr: the MAC address entry being added 6155 * @vid: VLAN ID 6156 * @flags: instructions from stack about fdb operation 6157 * @notified: whether notification was emitted 6158 * @extack: netlink extended ack 6159 */ 6160static int 6161ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 6162 struct net_device *dev, const unsigned char *addr, u16 vid, 6163 u16 flags, bool *notified, 6164 struct netlink_ext_ack __always_unused *extack) 6165{ 6166 int err; 6167 6168 if (vid) { 6169 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 6170 return -EINVAL; 6171 } 6172 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 6173 netdev_err(dev, "FDB only supports static addresses\n"); 6174 return -EINVAL; 6175 } 6176 6177 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 6178 err = dev_uc_add_excl(dev, addr); 6179 else if (is_multicast_ether_addr(addr)) 6180 err = dev_mc_add_excl(dev, addr); 6181 else 6182 err = -EINVAL; 6183 6184 /* Only return duplicate errors if NLM_F_EXCL is set */ 6185 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 6186 err = 0; 6187 6188 return err; 6189} 6190 6191/** 6192 * ice_fdb_del - delete an entry from the hardware database 6193 * @ndm: the input from the stack 6194 * @tb: pointer to array of nladdr (unused) 6195 * @dev: the net device pointer 6196 * @addr: the MAC address entry being added 6197 * @vid: VLAN ID 6198 * @notified: whether notification was emitted 6199 * @extack: netlink extended ack 6200 */ 6201static int 6202ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 6203 struct net_device *dev, const unsigned char *addr, 6204 __always_unused u16 vid, bool *notified, 6205 struct netlink_ext_ack *extack) 6206{ 6207 int err; 6208 6209 if (ndm->ndm_state & NUD_PERMANENT) { 6210 netdev_err(dev, "FDB only supports static addresses\n"); 6211 return -EINVAL; 6212 } 6213 6214 if (is_unicast_ether_addr(addr)) 6215 err = dev_uc_del(dev, addr); 6216 else if (is_multicast_ether_addr(addr)) 6217 err = dev_mc_del(dev, addr); 6218 else 6219 err = -EINVAL; 6220 6221 return err; 6222} 6223 6224#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 6225 NETIF_F_HW_VLAN_CTAG_TX | \ 6226 NETIF_F_HW_VLAN_STAG_RX | \ 6227 NETIF_F_HW_VLAN_STAG_TX) 6228 6229#define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 6230 NETIF_F_HW_VLAN_STAG_RX) 6231 6232#define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ 6233 NETIF_F_HW_VLAN_STAG_FILTER) 6234 6235/** 6236 * ice_fix_features - fix the netdev features flags based on device limitations 6237 * @netdev: ptr to the netdev that flags are being fixed on 6238 * @features: features that need to be checked and possibly fixed 6239 * 6240 * Make sure any fixups are made to features in this callback. This enables the 6241 * driver to not have to check unsupported configurations throughout the driver 6242 * because that's the responsiblity of this callback. 6243 * 6244 * Single VLAN Mode (SVM) Supported Features: 6245 * NETIF_F_HW_VLAN_CTAG_FILTER 6246 * NETIF_F_HW_VLAN_CTAG_RX 6247 * NETIF_F_HW_VLAN_CTAG_TX 6248 * 6249 * Double VLAN Mode (DVM) Supported Features: 6250 * NETIF_F_HW_VLAN_CTAG_FILTER 6251 * NETIF_F_HW_VLAN_CTAG_RX 6252 * NETIF_F_HW_VLAN_CTAG_TX 6253 * 6254 * NETIF_F_HW_VLAN_STAG_FILTER 6255 * NETIF_HW_VLAN_STAG_RX 6256 * NETIF_HW_VLAN_STAG_TX 6257 * 6258 * Features that need fixing: 6259 * Cannot simultaneously enable CTAG and STAG stripping and/or insertion. 6260 * These are mutually exlusive as the VSI context cannot support multiple 6261 * VLAN ethertypes simultaneously for stripping and/or insertion. If this 6262 * is not done, then default to clearing the requested STAG offload 6263 * settings. 6264 * 6265 * All supported filtering has to be enabled or disabled together. For 6266 * example, in DVM, CTAG and STAG filtering have to be enabled and disabled 6267 * together. If this is not done, then default to VLAN filtering disabled. 6268 * These are mutually exclusive as there is currently no way to 6269 * enable/disable VLAN filtering based on VLAN ethertype when using VLAN 6270 * prune rules. 6271 */ 6272static netdev_features_t 6273ice_fix_features(struct net_device *netdev, netdev_features_t features) 6274{ 6275 struct ice_netdev_priv *np = netdev_priv(netdev); 6276 netdev_features_t req_vlan_fltr, cur_vlan_fltr; 6277 bool cur_ctag, cur_stag, req_ctag, req_stag; 6278 6279 cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES; 6280 cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 6281 cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 6282 6283 req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES; 6284 req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 6285 req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 6286 6287 if (req_vlan_fltr != cur_vlan_fltr) { 6288 if (ice_is_dvm_ena(&np->vsi->back->hw)) { 6289 if (req_ctag && req_stag) { 6290 features |= NETIF_VLAN_FILTERING_FEATURES; 6291 } else if (!req_ctag && !req_stag) { 6292 features &= ~NETIF_VLAN_FILTERING_FEATURES; 6293 } else if ((!cur_ctag && req_ctag && !cur_stag) || 6294 (!cur_stag && req_stag && !cur_ctag)) { 6295 features |= NETIF_VLAN_FILTERING_FEATURES; 6296 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n"); 6297 } else if ((cur_ctag && !req_ctag && cur_stag) || 6298 (cur_stag && !req_stag && cur_ctag)) { 6299 features &= ~NETIF_VLAN_FILTERING_FEATURES; 6300 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n"); 6301 } 6302 } else { 6303 if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER) 6304 netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n"); 6305 6306 if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER) 6307 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 6308 } 6309 } 6310 6311 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 6312 (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) { 6313 netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 6314 features &= ~(NETIF_F_HW_VLAN_STAG_RX | 6315 NETIF_F_HW_VLAN_STAG_TX); 6316 } 6317 6318 if (!(netdev->features & NETIF_F_RXFCS) && 6319 (features & NETIF_F_RXFCS) && 6320 (features & NETIF_VLAN_STRIPPING_FEATURES) && 6321 !ice_vsi_has_non_zero_vlans(np->vsi)) { 6322 netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n"); 6323 features &= ~NETIF_VLAN_STRIPPING_FEATURES; 6324 } 6325 6326 return features; 6327} 6328 6329/** 6330 * ice_set_rx_rings_vlan_proto - update rings with new stripped VLAN proto 6331 * @vsi: PF's VSI 6332 * @vlan_ethertype: VLAN ethertype (802.1Q or 802.1ad) in network byte order 6333 * 6334 * Store current stripped VLAN proto in ring packet context, 6335 * so it can be accessed more efficiently by packet processing code. 6336 */ 6337static void 6338ice_set_rx_rings_vlan_proto(struct ice_vsi *vsi, __be16 vlan_ethertype) 6339{ 6340 u16 i; 6341 6342 ice_for_each_alloc_rxq(vsi, i) 6343 vsi->rx_rings[i]->pkt_ctx.vlan_proto = vlan_ethertype; 6344} 6345 6346/** 6347 * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI 6348 * @vsi: PF's VSI 6349 * @features: features used to determine VLAN offload settings 6350 * 6351 * First, determine the vlan_ethertype based on the VLAN offload bits in 6352 * features. Then determine if stripping and insertion should be enabled or 6353 * disabled. Finally enable or disable VLAN stripping and insertion. 6354 */ 6355static int 6356ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features) 6357{ 6358 bool enable_stripping = true, enable_insertion = true; 6359 struct ice_vsi_vlan_ops *vlan_ops; 6360 int strip_err = 0, insert_err = 0; 6361 u16 vlan_ethertype = 0; 6362 6363 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 6364 6365 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 6366 vlan_ethertype = ETH_P_8021AD; 6367 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 6368 vlan_ethertype = ETH_P_8021Q; 6369 6370 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 6371 enable_stripping = false; 6372 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 6373 enable_insertion = false; 6374 6375 if (enable_stripping) 6376 strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype); 6377 else 6378 strip_err = vlan_ops->dis_stripping(vsi); 6379 6380 if (enable_insertion) 6381 insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype); 6382 else 6383 insert_err = vlan_ops->dis_insertion(vsi); 6384 6385 if (strip_err || insert_err) 6386 return -EIO; 6387 6388 ice_set_rx_rings_vlan_proto(vsi, enable_stripping ? 6389 htons(vlan_ethertype) : 0); 6390 6391 return 0; 6392} 6393 6394/** 6395 * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI 6396 * @vsi: PF's VSI 6397 * @features: features used to determine VLAN filtering settings 6398 * 6399 * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the 6400 * features. 6401 */ 6402static int 6403ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features) 6404{ 6405 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 6406 int err = 0; 6407 6408 /* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking 6409 * if either bit is set. In switchdev mode Rx filtering should never be 6410 * enabled. 6411 */ 6412 if ((features & 6413 (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER)) && 6414 !ice_is_eswitch_mode_switchdev(vsi->back)) 6415 err = vlan_ops->ena_rx_filtering(vsi); 6416 else 6417 err = vlan_ops->dis_rx_filtering(vsi); 6418 6419 return err; 6420} 6421 6422/** 6423 * ice_set_vlan_features - set VLAN settings based on suggested feature set 6424 * @netdev: ptr to the netdev being adjusted 6425 * @features: the feature set that the stack is suggesting 6426 * 6427 * Only update VLAN settings if the requested_vlan_features are different than 6428 * the current_vlan_features. 6429 */ 6430static int 6431ice_set_vlan_features(struct net_device *netdev, netdev_features_t features) 6432{ 6433 netdev_features_t current_vlan_features, requested_vlan_features; 6434 struct ice_netdev_priv *np = netdev_priv(netdev); 6435 struct ice_vsi *vsi = np->vsi; 6436 int err; 6437 6438 current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES; 6439 requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES; 6440 if (current_vlan_features ^ requested_vlan_features) { 6441 if ((features & NETIF_F_RXFCS) && 6442 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 6443 dev_err(ice_pf_to_dev(vsi->back), 6444 "To enable VLAN stripping, you must first enable FCS/CRC stripping\n"); 6445 return -EIO; 6446 } 6447 6448 err = ice_set_vlan_offload_features(vsi, features); 6449 if (err) 6450 return err; 6451 } 6452 6453 current_vlan_features = netdev->features & 6454 NETIF_VLAN_FILTERING_FEATURES; 6455 requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES; 6456 if (current_vlan_features ^ requested_vlan_features) { 6457 err = ice_set_vlan_filtering_features(vsi, features); 6458 if (err) 6459 return err; 6460 } 6461 6462 return 0; 6463} 6464 6465/** 6466 * ice_set_loopback - turn on/off loopback mode on underlying PF 6467 * @vsi: ptr to VSI 6468 * @ena: flag to indicate the on/off setting 6469 */ 6470static int ice_set_loopback(struct ice_vsi *vsi, bool ena) 6471{ 6472 bool if_running = netif_running(vsi->netdev); 6473 int ret; 6474 6475 if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 6476 ret = ice_down(vsi); 6477 if (ret) { 6478 netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n"); 6479 return ret; 6480 } 6481 } 6482 ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 6483 if (ret) 6484 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 6485 if (if_running) 6486 ret = ice_up(vsi); 6487 6488 return ret; 6489} 6490 6491/** 6492 * ice_set_features - set the netdev feature flags 6493 * @netdev: ptr to the netdev being adjusted 6494 * @features: the feature set that the stack is suggesting 6495 */ 6496static int 6497ice_set_features(struct net_device *netdev, netdev_features_t features) 6498{ 6499 netdev_features_t changed = netdev->features ^ features; 6500 struct ice_netdev_priv *np = netdev_priv(netdev); 6501 struct ice_vsi *vsi = np->vsi; 6502 struct ice_pf *pf = vsi->back; 6503 int ret = 0; 6504 6505 /* Don't set any netdev advanced features with device in Safe Mode */ 6506 if (ice_is_safe_mode(pf)) { 6507 dev_err(ice_pf_to_dev(pf), 6508 "Device is in Safe Mode - not enabling advanced netdev features\n"); 6509 return ret; 6510 } 6511 6512 /* Do not change setting during reset */ 6513 if (ice_is_reset_in_progress(pf->state)) { 6514 dev_err(ice_pf_to_dev(pf), 6515 "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 6516 return -EBUSY; 6517 } 6518 6519 /* Multiple features can be changed in one call so keep features in 6520 * separate if/else statements to guarantee each feature is checked 6521 */ 6522 if (changed & NETIF_F_RXHASH) 6523 ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH)); 6524 6525 ret = ice_set_vlan_features(netdev, features); 6526 if (ret) 6527 return ret; 6528 6529 /* Turn on receive of FCS aka CRC, and after setting this 6530 * flag the packet data will have the 4 byte CRC appended 6531 */ 6532 if (changed & NETIF_F_RXFCS) { 6533 if ((features & NETIF_F_RXFCS) && 6534 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 6535 dev_err(ice_pf_to_dev(vsi->back), 6536 "To disable FCS/CRC stripping, you must first disable VLAN stripping\n"); 6537 return -EIO; 6538 } 6539 6540 ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS)); 6541 ret = ice_down_up(vsi); 6542 if (ret) 6543 return ret; 6544 } 6545 6546 if (changed & NETIF_F_NTUPLE) { 6547 bool ena = !!(features & NETIF_F_NTUPLE); 6548 6549 ice_vsi_manage_fdir(vsi, ena); 6550 ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi); 6551 } 6552 6553 /* don't turn off hw_tc_offload when ADQ is already enabled */ 6554 if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) { 6555 dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n"); 6556 return -EACCES; 6557 } 6558 6559 if (changed & NETIF_F_HW_TC) { 6560 bool ena = !!(features & NETIF_F_HW_TC); 6561 6562 assign_bit(ICE_FLAG_CLS_FLOWER, pf->flags, ena); 6563 } 6564 6565 if (changed & NETIF_F_LOOPBACK) 6566 ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 6567 6568 /* Due to E830 hardware limitations, TSO (NETIF_F_ALL_TSO) with GCS 6569 * (NETIF_F_HW_CSUM) is not supported. 6570 */ 6571 if (ice_is_feature_supported(pf, ICE_F_GCS) && 6572 ((features & NETIF_F_HW_CSUM) && (features & NETIF_F_ALL_TSO))) { 6573 if (netdev->features & NETIF_F_HW_CSUM) 6574 dev_err(ice_pf_to_dev(pf), "To enable TSO, you must first disable HW checksum.\n"); 6575 else 6576 dev_err(ice_pf_to_dev(pf), "To enable HW checksum, you must first disable TSO.\n"); 6577 return -EIO; 6578 } 6579 6580 return ret; 6581} 6582 6583/** 6584 * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI 6585 * @vsi: VSI to setup VLAN properties for 6586 */ 6587static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 6588{ 6589 int err; 6590 6591 err = ice_set_vlan_offload_features(vsi, vsi->netdev->features); 6592 if (err) 6593 return err; 6594 6595 err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features); 6596 if (err) 6597 return err; 6598 6599 return ice_vsi_add_vlan_zero(vsi); 6600} 6601 6602/** 6603 * ice_vsi_cfg_lan - Setup the VSI lan related config 6604 * @vsi: the VSI being configured 6605 * 6606 * Return 0 on success and negative value on error 6607 */ 6608int ice_vsi_cfg_lan(struct ice_vsi *vsi) 6609{ 6610 int err; 6611 6612 if (vsi->netdev && vsi->type == ICE_VSI_PF) { 6613 ice_set_rx_mode(vsi->netdev); 6614 6615 err = ice_vsi_vlan_setup(vsi); 6616 if (err) 6617 return err; 6618 } 6619 ice_vsi_cfg_dcb_rings(vsi); 6620 6621 err = ice_vsi_cfg_lan_txqs(vsi); 6622 if (!err && ice_is_xdp_ena_vsi(vsi)) 6623 err = ice_vsi_cfg_xdp_txqs(vsi); 6624 if (!err) 6625 err = ice_vsi_cfg_rxqs(vsi); 6626 6627 return err; 6628} 6629 6630/* THEORY OF MODERATION: 6631 * The ice driver hardware works differently than the hardware that DIMLIB was 6632 * originally made for. ice hardware doesn't have packet count limits that 6633 * can trigger an interrupt, but it *does* have interrupt rate limit support, 6634 * which is hard-coded to a limit of 250,000 ints/second. 6635 * If not using dynamic moderation, the INTRL value can be modified 6636 * by ethtool rx-usecs-high. 6637 */ 6638struct ice_dim { 6639 /* the throttle rate for interrupts, basically worst case delay before 6640 * an initial interrupt fires, value is stored in microseconds. 6641 */ 6642 u16 itr; 6643}; 6644 6645/* Make a different profile for Rx that doesn't allow quite so aggressive 6646 * moderation at the high end (it maxes out at 126us or about 8k interrupts a 6647 * second. 6648 */ 6649static const struct ice_dim rx_profile[] = { 6650 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6651 {8}, /* 125,000 ints/s */ 6652 {16}, /* 62,500 ints/s */ 6653 {62}, /* 16,129 ints/s */ 6654 {126} /* 7,936 ints/s */ 6655}; 6656 6657/* The transmit profile, which has the same sorts of values 6658 * as the previous struct 6659 */ 6660static const struct ice_dim tx_profile[] = { 6661 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6662 {8}, /* 125,000 ints/s */ 6663 {40}, /* 16,125 ints/s */ 6664 {128}, /* 7,812 ints/s */ 6665 {256} /* 3,906 ints/s */ 6666}; 6667 6668static void ice_tx_dim_work(struct work_struct *work) 6669{ 6670 struct ice_ring_container *rc; 6671 struct dim *dim; 6672 u16 itr; 6673 6674 dim = container_of(work, struct dim, work); 6675 rc = dim->priv; 6676 6677 WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile)); 6678 6679 /* look up the values in our local table */ 6680 itr = tx_profile[dim->profile_ix].itr; 6681 6682 ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim); 6683 ice_write_itr(rc, itr); 6684 6685 dim->state = DIM_START_MEASURE; 6686} 6687 6688static void ice_rx_dim_work(struct work_struct *work) 6689{ 6690 struct ice_ring_container *rc; 6691 struct dim *dim; 6692 u16 itr; 6693 6694 dim = container_of(work, struct dim, work); 6695 rc = dim->priv; 6696 6697 WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile)); 6698 6699 /* look up the values in our local table */ 6700 itr = rx_profile[dim->profile_ix].itr; 6701 6702 ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim); 6703 ice_write_itr(rc, itr); 6704 6705 dim->state = DIM_START_MEASURE; 6706} 6707 6708#define ICE_DIM_DEFAULT_PROFILE_IX 1 6709 6710/** 6711 * ice_init_moderation - set up interrupt moderation 6712 * @q_vector: the vector containing rings to be configured 6713 * 6714 * Set up interrupt moderation registers, with the intent to do the right thing 6715 * when called from reset or from probe, and whether or not dynamic moderation 6716 * is enabled or not. Take special care to write all the registers in both 6717 * dynamic moderation mode or not in order to make sure hardware is in a known 6718 * state. 6719 */ 6720static void ice_init_moderation(struct ice_q_vector *q_vector) 6721{ 6722 struct ice_ring_container *rc; 6723 bool tx_dynamic, rx_dynamic; 6724 6725 rc = &q_vector->tx; 6726 INIT_WORK(&rc->dim.work, ice_tx_dim_work); 6727 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6728 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6729 rc->dim.priv = rc; 6730 tx_dynamic = ITR_IS_DYNAMIC(rc); 6731 6732 /* set the initial TX ITR to match the above */ 6733 ice_write_itr(rc, tx_dynamic ? 6734 tx_profile[rc->dim.profile_ix].itr : rc->itr_setting); 6735 6736 rc = &q_vector->rx; 6737 INIT_WORK(&rc->dim.work, ice_rx_dim_work); 6738 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6739 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6740 rc->dim.priv = rc; 6741 rx_dynamic = ITR_IS_DYNAMIC(rc); 6742 6743 /* set the initial RX ITR to match the above */ 6744 ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr : 6745 rc->itr_setting); 6746 6747 ice_set_q_vector_intrl(q_vector); 6748} 6749 6750/** 6751 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 6752 * @vsi: the VSI being configured 6753 */ 6754static void ice_napi_enable_all(struct ice_vsi *vsi) 6755{ 6756 int q_idx; 6757 6758 if (!vsi->netdev) 6759 return; 6760 6761 ice_for_each_q_vector(vsi, q_idx) { 6762 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 6763 6764 ice_init_moderation(q_vector); 6765 6766 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 6767 napi_enable(&q_vector->napi); 6768 } 6769} 6770 6771/** 6772 * ice_up_complete - Finish the last steps of bringing up a connection 6773 * @vsi: The VSI being configured 6774 * 6775 * Return 0 on success and negative value on error 6776 */ 6777static int ice_up_complete(struct ice_vsi *vsi) 6778{ 6779 struct ice_pf *pf = vsi->back; 6780 int err; 6781 6782 ice_vsi_cfg_msix(vsi); 6783 6784 /* Enable only Rx rings, Tx rings were enabled by the FW when the 6785 * Tx queue group list was configured and the context bits were 6786 * programmed using ice_vsi_cfg_txqs 6787 */ 6788 err = ice_vsi_start_all_rx_rings(vsi); 6789 if (err) 6790 return err; 6791 6792 clear_bit(ICE_VSI_DOWN, vsi->state); 6793 ice_napi_enable_all(vsi); 6794 ice_vsi_ena_irq(vsi); 6795 6796 if (vsi->port_info && 6797 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 6798 ((vsi->netdev && (vsi->type == ICE_VSI_PF || 6799 vsi->type == ICE_VSI_SF)))) { 6800 ice_print_link_msg(vsi, true); 6801 netif_tx_start_all_queues(vsi->netdev); 6802 netif_carrier_on(vsi->netdev); 6803 ice_ptp_link_change(pf, true); 6804 } 6805 6806 /* Perform an initial read of the statistics registers now to 6807 * set the baseline so counters are ready when interface is up 6808 */ 6809 ice_update_eth_stats(vsi); 6810 6811 if (vsi->type == ICE_VSI_PF) 6812 ice_service_task_schedule(pf); 6813 6814 return 0; 6815} 6816 6817/** 6818 * ice_up - Bring the connection back up after being down 6819 * @vsi: VSI being configured 6820 */ 6821int ice_up(struct ice_vsi *vsi) 6822{ 6823 int err; 6824 6825 err = ice_vsi_cfg_lan(vsi); 6826 if (!err) 6827 err = ice_up_complete(vsi); 6828 6829 return err; 6830} 6831 6832/** 6833 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 6834 * @syncp: pointer to u64_stats_sync 6835 * @stats: stats that pkts and bytes count will be taken from 6836 * @pkts: packets stats counter 6837 * @bytes: bytes stats counter 6838 * 6839 * This function fetches stats from the ring considering the atomic operations 6840 * that needs to be performed to read u64 values in 32 bit machine. 6841 */ 6842void 6843ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, 6844 struct ice_q_stats stats, u64 *pkts, u64 *bytes) 6845{ 6846 unsigned int start; 6847 6848 do { 6849 start = u64_stats_fetch_begin(syncp); 6850 *pkts = stats.pkts; 6851 *bytes = stats.bytes; 6852 } while (u64_stats_fetch_retry(syncp, start)); 6853} 6854 6855/** 6856 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 6857 * @vsi: the VSI to be updated 6858 * @vsi_stats: the stats struct to be updated 6859 * @rings: rings to work on 6860 * @count: number of rings 6861 */ 6862static void 6863ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, 6864 struct rtnl_link_stats64 *vsi_stats, 6865 struct ice_tx_ring **rings, u16 count) 6866{ 6867 u16 i; 6868 6869 for (i = 0; i < count; i++) { 6870 struct ice_tx_ring *ring; 6871 u64 pkts = 0, bytes = 0; 6872 6873 ring = READ_ONCE(rings[i]); 6874 if (!ring || !ring->ring_stats) 6875 continue; 6876 ice_fetch_u64_stats_per_ring(&ring->ring_stats->syncp, 6877 ring->ring_stats->stats, &pkts, 6878 &bytes); 6879 vsi_stats->tx_packets += pkts; 6880 vsi_stats->tx_bytes += bytes; 6881 vsi->tx_restart += ring->ring_stats->tx_stats.restart_q; 6882 vsi->tx_busy += ring->ring_stats->tx_stats.tx_busy; 6883 vsi->tx_linearize += ring->ring_stats->tx_stats.tx_linearize; 6884 } 6885} 6886 6887/** 6888 * ice_update_vsi_ring_stats - Update VSI stats counters 6889 * @vsi: the VSI to be updated 6890 */ 6891static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 6892{ 6893 struct rtnl_link_stats64 *net_stats, *stats_prev; 6894 struct rtnl_link_stats64 *vsi_stats; 6895 struct ice_pf *pf = vsi->back; 6896 u64 pkts, bytes; 6897 int i; 6898 6899 vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC); 6900 if (!vsi_stats) 6901 return; 6902 6903 /* reset non-netdev (extended) stats */ 6904 vsi->tx_restart = 0; 6905 vsi->tx_busy = 0; 6906 vsi->tx_linearize = 0; 6907 vsi->rx_buf_failed = 0; 6908 vsi->rx_page_failed = 0; 6909 6910 rcu_read_lock(); 6911 6912 /* update Tx rings counters */ 6913 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings, 6914 vsi->num_txq); 6915 6916 /* update Rx rings counters */ 6917 ice_for_each_rxq(vsi, i) { 6918 struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]); 6919 struct ice_ring_stats *ring_stats; 6920 6921 ring_stats = ring->ring_stats; 6922 ice_fetch_u64_stats_per_ring(&ring_stats->syncp, 6923 ring_stats->stats, &pkts, 6924 &bytes); 6925 vsi_stats->rx_packets += pkts; 6926 vsi_stats->rx_bytes += bytes; 6927 vsi->rx_buf_failed += ring_stats->rx_stats.alloc_buf_failed; 6928 vsi->rx_page_failed += ring_stats->rx_stats.alloc_page_failed; 6929 } 6930 6931 /* update XDP Tx rings counters */ 6932 if (ice_is_xdp_ena_vsi(vsi)) 6933 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings, 6934 vsi->num_xdp_txq); 6935 6936 rcu_read_unlock(); 6937 6938 net_stats = &vsi->net_stats; 6939 stats_prev = &vsi->net_stats_prev; 6940 6941 /* Update netdev counters, but keep in mind that values could start at 6942 * random value after PF reset. And as we increase the reported stat by 6943 * diff of Prev-Cur, we need to be sure that Prev is valid. If it's not, 6944 * let's skip this round. 6945 */ 6946 if (likely(pf->stat_prev_loaded)) { 6947 net_stats->tx_packets += vsi_stats->tx_packets - stats_prev->tx_packets; 6948 net_stats->tx_bytes += vsi_stats->tx_bytes - stats_prev->tx_bytes; 6949 net_stats->rx_packets += vsi_stats->rx_packets - stats_prev->rx_packets; 6950 net_stats->rx_bytes += vsi_stats->rx_bytes - stats_prev->rx_bytes; 6951 } 6952 6953 stats_prev->tx_packets = vsi_stats->tx_packets; 6954 stats_prev->tx_bytes = vsi_stats->tx_bytes; 6955 stats_prev->rx_packets = vsi_stats->rx_packets; 6956 stats_prev->rx_bytes = vsi_stats->rx_bytes; 6957 6958 kfree(vsi_stats); 6959} 6960 6961/** 6962 * ice_update_vsi_stats - Update VSI stats counters 6963 * @vsi: the VSI to be updated 6964 */ 6965void ice_update_vsi_stats(struct ice_vsi *vsi) 6966{ 6967 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 6968 struct ice_eth_stats *cur_es = &vsi->eth_stats; 6969 struct ice_pf *pf = vsi->back; 6970 6971 if (test_bit(ICE_VSI_DOWN, vsi->state) || 6972 test_bit(ICE_CFG_BUSY, pf->state)) 6973 return; 6974 6975 /* get stats as recorded by Tx/Rx rings */ 6976 ice_update_vsi_ring_stats(vsi); 6977 6978 /* get VSI stats as recorded by the hardware */ 6979 ice_update_eth_stats(vsi); 6980 6981 cur_ns->tx_errors = cur_es->tx_errors; 6982 cur_ns->rx_dropped = cur_es->rx_discards; 6983 cur_ns->tx_dropped = cur_es->tx_discards; 6984 cur_ns->multicast = cur_es->rx_multicast; 6985 6986 /* update some more netdev stats if this is main VSI */ 6987 if (vsi->type == ICE_VSI_PF) { 6988 cur_ns->rx_crc_errors = pf->stats.crc_errors; 6989 cur_ns->rx_errors = pf->stats.crc_errors + 6990 pf->stats.illegal_bytes + 6991 pf->stats.rx_undersize + 6992 pf->stats.rx_jabber + 6993 pf->stats.rx_fragments + 6994 pf->stats.rx_oversize; 6995 /* record drops from the port level */ 6996 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 6997 } 6998} 6999 7000/** 7001 * ice_update_pf_stats - Update PF port stats counters 7002 * @pf: PF whose stats needs to be updated 7003 */ 7004void ice_update_pf_stats(struct ice_pf *pf) 7005{ 7006 struct ice_hw_port_stats *prev_ps, *cur_ps; 7007 struct ice_hw *hw = &pf->hw; 7008 u16 fd_ctr_base; 7009 u8 port; 7010 7011 port = hw->port_info->lport; 7012 prev_ps = &pf->stats_prev; 7013 cur_ps = &pf->stats; 7014 7015 if (ice_is_reset_in_progress(pf->state)) 7016 pf->stat_prev_loaded = false; 7017 7018 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 7019 &prev_ps->eth.rx_bytes, 7020 &cur_ps->eth.rx_bytes); 7021 7022 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 7023 &prev_ps->eth.rx_unicast, 7024 &cur_ps->eth.rx_unicast); 7025 7026 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 7027 &prev_ps->eth.rx_multicast, 7028 &cur_ps->eth.rx_multicast); 7029 7030 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 7031 &prev_ps->eth.rx_broadcast, 7032 &cur_ps->eth.rx_broadcast); 7033 7034 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 7035 &prev_ps->eth.rx_discards, 7036 &cur_ps->eth.rx_discards); 7037 7038 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 7039 &prev_ps->eth.tx_bytes, 7040 &cur_ps->eth.tx_bytes); 7041 7042 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 7043 &prev_ps->eth.tx_unicast, 7044 &cur_ps->eth.tx_unicast); 7045 7046 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 7047 &prev_ps->eth.tx_multicast, 7048 &cur_ps->eth.tx_multicast); 7049 7050 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 7051 &prev_ps->eth.tx_broadcast, 7052 &cur_ps->eth.tx_broadcast); 7053 7054 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 7055 &prev_ps->tx_dropped_link_down, 7056 &cur_ps->tx_dropped_link_down); 7057 7058 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 7059 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 7060 7061 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 7062 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 7063 7064 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 7065 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 7066 7067 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 7068 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 7069 7070 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 7071 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 7072 7073 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 7074 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 7075 7076 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 7077 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 7078 7079 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 7080 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 7081 7082 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 7083 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 7084 7085 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 7086 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 7087 7088 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 7089 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 7090 7091 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 7092 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 7093 7094 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 7095 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 7096 7097 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 7098 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 7099 7100 fd_ctr_base = hw->fd_ctr_base; 7101 7102 ice_stat_update40(hw, 7103 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 7104 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 7105 &cur_ps->fd_sb_match); 7106 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 7107 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 7108 7109 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 7110 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 7111 7112 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 7113 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 7114 7115 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 7116 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 7117 7118 ice_update_dcb_stats(pf); 7119 7120 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 7121 &prev_ps->crc_errors, &cur_ps->crc_errors); 7122 7123 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 7124 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 7125 7126 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 7127 &prev_ps->mac_local_faults, 7128 &cur_ps->mac_local_faults); 7129 7130 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 7131 &prev_ps->mac_remote_faults, 7132 &cur_ps->mac_remote_faults); 7133 7134 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 7135 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 7136 7137 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 7138 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 7139 7140 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 7141 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 7142 7143 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 7144 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 7145 7146 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 7147 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 7148 7149 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 7150 7151 pf->stat_prev_loaded = true; 7152} 7153 7154/** 7155 * ice_get_stats64 - get statistics for network device structure 7156 * @netdev: network interface device structure 7157 * @stats: main device statistics structure 7158 */ 7159void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 7160{ 7161 struct ice_netdev_priv *np = netdev_priv(netdev); 7162 struct rtnl_link_stats64 *vsi_stats; 7163 struct ice_vsi *vsi = np->vsi; 7164 7165 vsi_stats = &vsi->net_stats; 7166 7167 if (!vsi->num_txq || !vsi->num_rxq) 7168 return; 7169 7170 /* netdev packet/byte stats come from ring counter. These are obtained 7171 * by summing up ring counters (done by ice_update_vsi_ring_stats). 7172 * But, only call the update routine and read the registers if VSI is 7173 * not down. 7174 */ 7175 if (!test_bit(ICE_VSI_DOWN, vsi->state)) 7176 ice_update_vsi_ring_stats(vsi); 7177 stats->tx_packets = vsi_stats->tx_packets; 7178 stats->tx_bytes = vsi_stats->tx_bytes; 7179 stats->rx_packets = vsi_stats->rx_packets; 7180 stats->rx_bytes = vsi_stats->rx_bytes; 7181 7182 /* The rest of the stats can be read from the hardware but instead we 7183 * just return values that the watchdog task has already obtained from 7184 * the hardware. 7185 */ 7186 stats->multicast = vsi_stats->multicast; 7187 stats->tx_errors = vsi_stats->tx_errors; 7188 stats->tx_dropped = vsi_stats->tx_dropped; 7189 stats->rx_errors = vsi_stats->rx_errors; 7190 stats->rx_dropped = vsi_stats->rx_dropped; 7191 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 7192 stats->rx_length_errors = vsi_stats->rx_length_errors; 7193} 7194 7195/** 7196 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 7197 * @vsi: VSI having NAPI disabled 7198 */ 7199static void ice_napi_disable_all(struct ice_vsi *vsi) 7200{ 7201 int q_idx; 7202 7203 if (!vsi->netdev) 7204 return; 7205 7206 ice_for_each_q_vector(vsi, q_idx) { 7207 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 7208 7209 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 7210 napi_disable(&q_vector->napi); 7211 7212 cancel_work_sync(&q_vector->tx.dim.work); 7213 cancel_work_sync(&q_vector->rx.dim.work); 7214 } 7215} 7216 7217/** 7218 * ice_vsi_dis_irq - Mask off queue interrupt generation on the VSI 7219 * @vsi: the VSI being un-configured 7220 */ 7221static void ice_vsi_dis_irq(struct ice_vsi *vsi) 7222{ 7223 struct ice_pf *pf = vsi->back; 7224 struct ice_hw *hw = &pf->hw; 7225 u32 val; 7226 int i; 7227 7228 /* disable interrupt causation from each Rx queue; Tx queues are 7229 * handled in ice_vsi_stop_tx_ring() 7230 */ 7231 if (vsi->rx_rings) { 7232 ice_for_each_rxq(vsi, i) { 7233 if (vsi->rx_rings[i]) { 7234 u16 reg; 7235 7236 reg = vsi->rx_rings[i]->reg_idx; 7237 val = rd32(hw, QINT_RQCTL(reg)); 7238 val &= ~QINT_RQCTL_CAUSE_ENA_M; 7239 wr32(hw, QINT_RQCTL(reg), val); 7240 } 7241 } 7242 } 7243 7244 /* disable each interrupt */ 7245 ice_for_each_q_vector(vsi, i) { 7246 if (!vsi->q_vectors[i]) 7247 continue; 7248 wr32(hw, GLINT_DYN_CTL(vsi->q_vectors[i]->reg_idx), 0); 7249 } 7250 7251 ice_flush(hw); 7252 7253 /* don't call synchronize_irq() for VF's from the host */ 7254 if (vsi->type == ICE_VSI_VF) 7255 return; 7256 7257 ice_for_each_q_vector(vsi, i) 7258 synchronize_irq(vsi->q_vectors[i]->irq.virq); 7259} 7260 7261/** 7262 * ice_down - Shutdown the connection 7263 * @vsi: The VSI being stopped 7264 * 7265 * Caller of this function is expected to set the vsi->state ICE_DOWN bit 7266 */ 7267int ice_down(struct ice_vsi *vsi) 7268{ 7269 int i, tx_err, rx_err, vlan_err = 0; 7270 7271 WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state)); 7272 7273 if (vsi->netdev) { 7274 vlan_err = ice_vsi_del_vlan_zero(vsi); 7275 ice_ptp_link_change(vsi->back, false); 7276 netif_carrier_off(vsi->netdev); 7277 netif_tx_disable(vsi->netdev); 7278 } 7279 7280 ice_vsi_dis_irq(vsi); 7281 7282 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 7283 if (tx_err) 7284 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 7285 vsi->vsi_num, tx_err); 7286 if (!tx_err && vsi->xdp_rings) { 7287 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 7288 if (tx_err) 7289 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 7290 vsi->vsi_num, tx_err); 7291 } 7292 7293 rx_err = ice_vsi_stop_all_rx_rings(vsi); 7294 if (rx_err) 7295 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 7296 vsi->vsi_num, rx_err); 7297 7298 ice_napi_disable_all(vsi); 7299 7300 ice_for_each_txq(vsi, i) 7301 ice_clean_tx_ring(vsi->tx_rings[i]); 7302 7303 if (vsi->xdp_rings) 7304 ice_for_each_xdp_txq(vsi, i) 7305 ice_clean_tx_ring(vsi->xdp_rings[i]); 7306 7307 ice_for_each_rxq(vsi, i) 7308 ice_clean_rx_ring(vsi->rx_rings[i]); 7309 7310 if (tx_err || rx_err || vlan_err) { 7311 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 7312 vsi->vsi_num, vsi->vsw->sw_id); 7313 return -EIO; 7314 } 7315 7316 return 0; 7317} 7318 7319/** 7320 * ice_down_up - shutdown the VSI connection and bring it up 7321 * @vsi: the VSI to be reconnected 7322 */ 7323int ice_down_up(struct ice_vsi *vsi) 7324{ 7325 int ret; 7326 7327 /* if DOWN already set, nothing to do */ 7328 if (test_and_set_bit(ICE_VSI_DOWN, vsi->state)) 7329 return 0; 7330 7331 ret = ice_down(vsi); 7332 if (ret) 7333 return ret; 7334 7335 ret = ice_up(vsi); 7336 if (ret) { 7337 netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n"); 7338 return ret; 7339 } 7340 7341 return 0; 7342} 7343 7344/** 7345 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 7346 * @vsi: VSI having resources allocated 7347 * 7348 * Return 0 on success, negative on failure 7349 */ 7350int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 7351{ 7352 int i, err = 0; 7353 7354 if (!vsi->num_txq) { 7355 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 7356 vsi->vsi_num); 7357 return -EINVAL; 7358 } 7359 7360 ice_for_each_txq(vsi, i) { 7361 struct ice_tx_ring *ring = vsi->tx_rings[i]; 7362 7363 if (!ring) 7364 return -EINVAL; 7365 7366 if (vsi->netdev) 7367 ring->netdev = vsi->netdev; 7368 err = ice_setup_tx_ring(ring); 7369 if (err) 7370 break; 7371 } 7372 7373 return err; 7374} 7375 7376/** 7377 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 7378 * @vsi: VSI having resources allocated 7379 * 7380 * Return 0 on success, negative on failure 7381 */ 7382int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 7383{ 7384 int i, err = 0; 7385 7386 if (!vsi->num_rxq) { 7387 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 7388 vsi->vsi_num); 7389 return -EINVAL; 7390 } 7391 7392 ice_for_each_rxq(vsi, i) { 7393 struct ice_rx_ring *ring = vsi->rx_rings[i]; 7394 7395 if (!ring) 7396 return -EINVAL; 7397 7398 if (vsi->netdev) 7399 ring->netdev = vsi->netdev; 7400 err = ice_setup_rx_ring(ring); 7401 if (err) 7402 break; 7403 } 7404 7405 return err; 7406} 7407 7408/** 7409 * ice_vsi_open_ctrl - open control VSI for use 7410 * @vsi: the VSI to open 7411 * 7412 * Initialization of the Control VSI 7413 * 7414 * Returns 0 on success, negative value on error 7415 */ 7416int ice_vsi_open_ctrl(struct ice_vsi *vsi) 7417{ 7418 char int_name[ICE_INT_NAME_STR_LEN]; 7419 struct ice_pf *pf = vsi->back; 7420 struct device *dev; 7421 int err; 7422 7423 dev = ice_pf_to_dev(pf); 7424 /* allocate descriptors */ 7425 err = ice_vsi_setup_tx_rings(vsi); 7426 if (err) 7427 goto err_setup_tx; 7428 7429 err = ice_vsi_setup_rx_rings(vsi); 7430 if (err) 7431 goto err_setup_rx; 7432 7433 err = ice_vsi_cfg_lan(vsi); 7434 if (err) 7435 goto err_setup_rx; 7436 7437 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 7438 dev_driver_string(dev), dev_name(dev)); 7439 err = ice_vsi_req_irq_msix(vsi, int_name); 7440 if (err) 7441 goto err_setup_rx; 7442 7443 ice_vsi_cfg_msix(vsi); 7444 7445 err = ice_vsi_start_all_rx_rings(vsi); 7446 if (err) 7447 goto err_up_complete; 7448 7449 clear_bit(ICE_VSI_DOWN, vsi->state); 7450 ice_vsi_ena_irq(vsi); 7451 7452 return 0; 7453 7454err_up_complete: 7455 ice_down(vsi); 7456err_setup_rx: 7457 ice_vsi_free_rx_rings(vsi); 7458err_setup_tx: 7459 ice_vsi_free_tx_rings(vsi); 7460 7461 return err; 7462} 7463 7464/** 7465 * ice_vsi_open - Called when a network interface is made active 7466 * @vsi: the VSI to open 7467 * 7468 * Initialization of the VSI 7469 * 7470 * Returns 0 on success, negative value on error 7471 */ 7472int ice_vsi_open(struct ice_vsi *vsi) 7473{ 7474 char int_name[ICE_INT_NAME_STR_LEN]; 7475 struct ice_pf *pf = vsi->back; 7476 int err; 7477 7478 /* allocate descriptors */ 7479 err = ice_vsi_setup_tx_rings(vsi); 7480 if (err) 7481 goto err_setup_tx; 7482 7483 err = ice_vsi_setup_rx_rings(vsi); 7484 if (err) 7485 goto err_setup_rx; 7486 7487 err = ice_vsi_cfg_lan(vsi); 7488 if (err) 7489 goto err_setup_rx; 7490 7491 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7492 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 7493 err = ice_vsi_req_irq_msix(vsi, int_name); 7494 if (err) 7495 goto err_setup_rx; 7496 7497 if (bitmap_empty(pf->txtime_txqs, pf->max_pf_txqs)) 7498 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 7499 7500 if (vsi->type == ICE_VSI_PF || vsi->type == ICE_VSI_SF) { 7501 /* Notify the stack of the actual queue counts. */ 7502 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 7503 if (err) 7504 goto err_set_qs; 7505 7506 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 7507 if (err) 7508 goto err_set_qs; 7509 7510 ice_vsi_set_napi_queues(vsi); 7511 } 7512 7513 err = ice_up_complete(vsi); 7514 if (err) 7515 goto err_up_complete; 7516 7517 return 0; 7518 7519err_up_complete: 7520 ice_down(vsi); 7521err_set_qs: 7522 ice_vsi_free_irq(vsi); 7523err_setup_rx: 7524 ice_vsi_free_rx_rings(vsi); 7525err_setup_tx: 7526 ice_vsi_free_tx_rings(vsi); 7527 7528 return err; 7529} 7530 7531/** 7532 * ice_vsi_release_all - Delete all VSIs 7533 * @pf: PF from which all VSIs are being removed 7534 */ 7535static void ice_vsi_release_all(struct ice_pf *pf) 7536{ 7537 int err, i; 7538 7539 if (!pf->vsi) 7540 return; 7541 7542 ice_for_each_vsi(pf, i) { 7543 if (!pf->vsi[i]) 7544 continue; 7545 7546 if (pf->vsi[i]->type == ICE_VSI_CHNL) 7547 continue; 7548 7549 err = ice_vsi_release(pf->vsi[i]); 7550 if (err) 7551 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 7552 i, err, pf->vsi[i]->vsi_num); 7553 } 7554} 7555 7556/** 7557 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 7558 * @pf: pointer to the PF instance 7559 * @type: VSI type to rebuild 7560 * 7561 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 7562 */ 7563static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 7564{ 7565 struct device *dev = ice_pf_to_dev(pf); 7566 int i, err; 7567 7568 ice_for_each_vsi(pf, i) { 7569 struct ice_vsi *vsi = pf->vsi[i]; 7570 7571 if (!vsi || vsi->type != type) 7572 continue; 7573 7574 /* rebuild the VSI */ 7575 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT); 7576 if (err) { 7577 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 7578 err, vsi->idx, ice_vsi_type_str(type)); 7579 return err; 7580 } 7581 7582 /* replay filters for the VSI */ 7583 err = ice_replay_vsi(&pf->hw, vsi->idx); 7584 if (err) { 7585 dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n", 7586 err, vsi->idx, ice_vsi_type_str(type)); 7587 return err; 7588 } 7589 7590 /* Re-map HW VSI number, using VSI handle that has been 7591 * previously validated in ice_replay_vsi() call above 7592 */ 7593 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 7594 7595 /* enable the VSI */ 7596 err = ice_ena_vsi(vsi, false); 7597 if (err) { 7598 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 7599 err, vsi->idx, ice_vsi_type_str(type)); 7600 return err; 7601 } 7602 7603 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 7604 ice_vsi_type_str(type)); 7605 } 7606 7607 return 0; 7608} 7609 7610/** 7611 * ice_update_pf_netdev_link - Update PF netdev link status 7612 * @pf: pointer to the PF instance 7613 */ 7614static void ice_update_pf_netdev_link(struct ice_pf *pf) 7615{ 7616 bool link_up; 7617 int i; 7618 7619 ice_for_each_vsi(pf, i) { 7620 struct ice_vsi *vsi = pf->vsi[i]; 7621 7622 if (!vsi || vsi->type != ICE_VSI_PF) 7623 return; 7624 7625 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 7626 if (link_up) { 7627 netif_carrier_on(pf->vsi[i]->netdev); 7628 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 7629 } else { 7630 netif_carrier_off(pf->vsi[i]->netdev); 7631 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 7632 } 7633 } 7634} 7635 7636/** 7637 * ice_rebuild - rebuild after reset 7638 * @pf: PF to rebuild 7639 * @reset_type: type of reset 7640 * 7641 * Do not rebuild VF VSI in this flow because that is already handled via 7642 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 7643 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 7644 * to reset/rebuild all the VF VSI twice. 7645 */ 7646static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 7647{ 7648 struct ice_vsi *vsi = ice_get_main_vsi(pf); 7649 struct device *dev = ice_pf_to_dev(pf); 7650 struct ice_hw *hw = &pf->hw; 7651 bool dvm; 7652 int err; 7653 7654 if (test_bit(ICE_DOWN, pf->state)) 7655 goto clear_recovery; 7656 7657 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 7658 7659#define ICE_EMP_RESET_SLEEP_MS 5000 7660 if (reset_type == ICE_RESET_EMPR) { 7661 /* If an EMP reset has occurred, any previously pending flash 7662 * update will have completed. We no longer know whether or 7663 * not the NVM update EMP reset is restricted. 7664 */ 7665 pf->fw_emp_reset_disabled = false; 7666 7667 msleep(ICE_EMP_RESET_SLEEP_MS); 7668 } 7669 7670 err = ice_init_all_ctrlq(hw); 7671 if (err) { 7672 dev_err(dev, "control queues init failed %d\n", err); 7673 goto err_init_ctrlq; 7674 } 7675 7676 /* if DDP was previously loaded successfully */ 7677 if (!ice_is_safe_mode(pf)) { 7678 /* reload the SW DB of filter tables */ 7679 if (reset_type == ICE_RESET_PFR) 7680 ice_fill_blk_tbls(hw); 7681 else 7682 /* Reload DDP Package after CORER/GLOBR reset */ 7683 ice_load_pkg(NULL, pf); 7684 } 7685 7686 err = ice_clear_pf_cfg(hw); 7687 if (err) { 7688 dev_err(dev, "clear PF configuration failed %d\n", err); 7689 goto err_init_ctrlq; 7690 } 7691 7692 ice_clear_pxe_mode(hw); 7693 7694 err = ice_init_nvm(hw); 7695 if (err) { 7696 dev_err(dev, "ice_init_nvm failed %d\n", err); 7697 goto err_init_ctrlq; 7698 } 7699 7700 err = ice_get_caps(hw); 7701 if (err) { 7702 dev_err(dev, "ice_get_caps failed %d\n", err); 7703 goto err_init_ctrlq; 7704 } 7705 7706 err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 7707 if (err) { 7708 dev_err(dev, "set_mac_cfg failed %d\n", err); 7709 goto err_init_ctrlq; 7710 } 7711 7712 dvm = ice_is_dvm_ena(hw); 7713 7714 err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 7715 if (err) 7716 goto err_init_ctrlq; 7717 7718 err = ice_sched_init_port(hw->port_info); 7719 if (err) 7720 goto err_sched_init_port; 7721 7722 /* start misc vector */ 7723 err = ice_req_irq_msix_misc(pf); 7724 if (err) { 7725 dev_err(dev, "misc vector setup failed: %d\n", err); 7726 goto err_sched_init_port; 7727 } 7728 7729 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7730 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 7731 if (!rd32(hw, PFQF_FD_SIZE)) { 7732 u16 unused, guar, b_effort; 7733 7734 guar = hw->func_caps.fd_fltr_guar; 7735 b_effort = hw->func_caps.fd_fltr_best_effort; 7736 7737 /* force guaranteed filter pool for PF */ 7738 ice_alloc_fd_guar_item(hw, &unused, guar); 7739 /* force shared filter pool for PF */ 7740 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 7741 } 7742 } 7743 7744 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 7745 ice_dcb_rebuild(pf); 7746 7747 /* If the PF previously had enabled PTP, PTP init needs to happen before 7748 * the VSI rebuild. If not, this causes the PTP link status events to 7749 * fail. 7750 */ 7751 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 7752 ice_ptp_rebuild(pf, reset_type); 7753 7754 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 7755 ice_gnss_init(pf); 7756 7757 /* rebuild PF VSI */ 7758 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 7759 if (err) { 7760 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 7761 goto err_vsi_rebuild; 7762 } 7763 7764 if (reset_type == ICE_RESET_PFR) { 7765 err = ice_rebuild_channels(pf); 7766 if (err) { 7767 dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n", 7768 err); 7769 goto err_vsi_rebuild; 7770 } 7771 } 7772 7773 /* If Flow Director is active */ 7774 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7775 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 7776 if (err) { 7777 dev_err(dev, "control VSI rebuild failed: %d\n", err); 7778 goto err_vsi_rebuild; 7779 } 7780 7781 /* replay HW Flow Director recipes */ 7782 if (hw->fdir_prof) 7783 ice_fdir_replay_flows(hw); 7784 7785 /* replay Flow Director filters */ 7786 ice_fdir_replay_fltrs(pf); 7787 7788 ice_rebuild_arfs(pf); 7789 } 7790 7791 if (vsi && vsi->netdev) 7792 netif_device_attach(vsi->netdev); 7793 7794 ice_update_pf_netdev_link(pf); 7795 7796 /* tell the firmware we are up */ 7797 err = ice_send_version(pf); 7798 if (err) { 7799 dev_err(dev, "Rebuild failed due to error sending driver version: %d\n", 7800 err); 7801 goto err_vsi_rebuild; 7802 } 7803 7804 ice_replay_post(hw); 7805 7806 /* if we get here, reset flow is successful */ 7807 clear_bit(ICE_RESET_FAILED, pf->state); 7808 7809 ice_health_clear(pf); 7810 7811 ice_rdma_finalize_setup(pf); 7812 if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG)) 7813 ice_lag_rebuild(pf); 7814 7815 /* Restore timestamp mode settings after VSI rebuild */ 7816 ice_ptp_restore_timestamp_mode(pf); 7817 7818 /* Start PTP periodic work after VSI is fully rebuilt */ 7819 ice_ptp_queue_work(pf); 7820 return; 7821 7822err_vsi_rebuild: 7823err_sched_init_port: 7824 ice_sched_cleanup_all(hw); 7825err_init_ctrlq: 7826 ice_shutdown_all_ctrlq(hw, false); 7827 set_bit(ICE_RESET_FAILED, pf->state); 7828clear_recovery: 7829 /* set this bit in PF state to control service task scheduling */ 7830 set_bit(ICE_NEEDS_RESTART, pf->state); 7831 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 7832} 7833 7834/** 7835 * ice_change_mtu - NDO callback to change the MTU 7836 * @netdev: network interface device structure 7837 * @new_mtu: new value for maximum frame size 7838 * 7839 * Returns 0 on success, negative on failure 7840 */ 7841int ice_change_mtu(struct net_device *netdev, int new_mtu) 7842{ 7843 struct ice_netdev_priv *np = netdev_priv(netdev); 7844 struct ice_vsi *vsi = np->vsi; 7845 struct ice_pf *pf = vsi->back; 7846 struct bpf_prog *prog; 7847 u8 count = 0; 7848 int err = 0; 7849 7850 if (new_mtu == (int)netdev->mtu) { 7851 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 7852 return 0; 7853 } 7854 7855 prog = vsi->xdp_prog; 7856 if (prog && !prog->aux->xdp_has_frags) { 7857 int frame_size = ice_max_xdp_frame_size(vsi); 7858 7859 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 7860 netdev_err(netdev, "max MTU for XDP usage is %d\n", 7861 frame_size - ICE_ETH_PKT_HDR_PAD); 7862 return -EINVAL; 7863 } 7864 } 7865 7866 /* if a reset is in progress, wait for some time for it to complete */ 7867 do { 7868 if (ice_is_reset_in_progress(pf->state)) { 7869 count++; 7870 usleep_range(1000, 2000); 7871 } else { 7872 break; 7873 } 7874 7875 } while (count < 100); 7876 7877 if (count == 100) { 7878 netdev_err(netdev, "can't change MTU. Device is busy\n"); 7879 return -EBUSY; 7880 } 7881 7882 WRITE_ONCE(netdev->mtu, (unsigned int)new_mtu); 7883 err = ice_down_up(vsi); 7884 if (err) 7885 return err; 7886 7887 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 7888 set_bit(ICE_FLAG_MTU_CHANGED, pf->flags); 7889 7890 return err; 7891} 7892 7893/** 7894 * ice_set_rss_lut - Set RSS LUT 7895 * @vsi: Pointer to VSI structure 7896 * @lut: Lookup table 7897 * @lut_size: Lookup table size 7898 * 7899 * Returns 0 on success, negative on failure 7900 */ 7901int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7902{ 7903 struct ice_aq_get_set_rss_lut_params params = {}; 7904 struct ice_hw *hw = &vsi->back->hw; 7905 int status; 7906 7907 if (!lut) 7908 return -EINVAL; 7909 7910 params.vsi_handle = vsi->idx; 7911 params.lut_size = lut_size; 7912 params.lut_type = vsi->rss_lut_type; 7913 params.lut = lut; 7914 7915 status = ice_aq_set_rss_lut(hw, ¶ms); 7916 if (status) 7917 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n", 7918 status, libie_aq_str(hw->adminq.sq_last_status)); 7919 7920 return status; 7921} 7922 7923/** 7924 * ice_set_rss_key - Set RSS key 7925 * @vsi: Pointer to the VSI structure 7926 * @seed: RSS hash seed 7927 * 7928 * Returns 0 on success, negative on failure 7929 */ 7930int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed) 7931{ 7932 struct ice_hw *hw = &vsi->back->hw; 7933 int status; 7934 7935 if (!seed) 7936 return -EINVAL; 7937 7938 status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7939 if (status) 7940 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n", 7941 status, libie_aq_str(hw->adminq.sq_last_status)); 7942 7943 return status; 7944} 7945 7946/** 7947 * ice_get_rss_lut - Get RSS LUT 7948 * @vsi: Pointer to VSI structure 7949 * @lut: Buffer to store the lookup table entries 7950 * @lut_size: Size of buffer to store the lookup table entries 7951 * 7952 * Returns 0 on success, negative on failure 7953 */ 7954int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7955{ 7956 struct ice_aq_get_set_rss_lut_params params = {}; 7957 struct ice_hw *hw = &vsi->back->hw; 7958 int status; 7959 7960 if (!lut) 7961 return -EINVAL; 7962 7963 params.vsi_handle = vsi->idx; 7964 params.lut_size = lut_size; 7965 params.lut_type = vsi->rss_lut_type; 7966 params.lut = lut; 7967 7968 status = ice_aq_get_rss_lut(hw, ¶ms); 7969 if (status) 7970 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n", 7971 status, libie_aq_str(hw->adminq.sq_last_status)); 7972 7973 return status; 7974} 7975 7976/** 7977 * ice_get_rss_key - Get RSS key 7978 * @vsi: Pointer to VSI structure 7979 * @seed: Buffer to store the key in 7980 * 7981 * Returns 0 on success, negative on failure 7982 */ 7983int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed) 7984{ 7985 struct ice_hw *hw = &vsi->back->hw; 7986 int status; 7987 7988 if (!seed) 7989 return -EINVAL; 7990 7991 status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7992 if (status) 7993 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n", 7994 status, libie_aq_str(hw->adminq.sq_last_status)); 7995 7996 return status; 7997} 7998 7999/** 8000 * ice_get_rss - Get RSS LUT and/or key 8001 * @vsi: Pointer to VSI structure 8002 * @seed: Buffer to store the key in 8003 * @lut: Buffer to store the lookup table entries 8004 * @lut_size: Size of buffer to store the lookup table entries 8005 * 8006 * Return: 0 on success, negative on failure 8007 */ 8008int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8009{ 8010 int err; 8011 8012 if (seed) { 8013 err = ice_get_rss_key(vsi, seed); 8014 if (err) 8015 return err; 8016 } 8017 8018 if (lut) { 8019 err = ice_get_rss_lut(vsi, lut, lut_size); 8020 if (err) 8021 return err; 8022 } 8023 8024 return 0; 8025} 8026 8027/** 8028 * ice_set_rss_hfunc - Set RSS HASH function 8029 * @vsi: Pointer to VSI structure 8030 * @hfunc: hash function (ICE_AQ_VSI_Q_OPT_RSS_*) 8031 * 8032 * Returns 0 on success, negative on failure 8033 */ 8034int ice_set_rss_hfunc(struct ice_vsi *vsi, u8 hfunc) 8035{ 8036 struct ice_hw *hw = &vsi->back->hw; 8037 struct ice_vsi_ctx *ctx; 8038 bool symm; 8039 int err; 8040 8041 if (hfunc == vsi->rss_hfunc) 8042 return 0; 8043 8044 if (hfunc != ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ && 8045 hfunc != ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ) 8046 return -EOPNOTSUPP; 8047 8048 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 8049 if (!ctx) 8050 return -ENOMEM; 8051 8052 ctx->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_Q_OPT_VALID); 8053 ctx->info.q_opt_rss = vsi->info.q_opt_rss; 8054 ctx->info.q_opt_rss &= ~ICE_AQ_VSI_Q_OPT_RSS_HASH_M; 8055 ctx->info.q_opt_rss |= 8056 FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_HASH_M, hfunc); 8057 ctx->info.q_opt_tc = vsi->info.q_opt_tc; 8058 ctx->info.q_opt_flags = vsi->info.q_opt_rss; 8059 8060 err = ice_update_vsi(hw, vsi->idx, ctx, NULL); 8061 if (err) { 8062 dev_err(ice_pf_to_dev(vsi->back), "Failed to configure RSS hash for VSI %d, error %d\n", 8063 vsi->vsi_num, err); 8064 } else { 8065 vsi->info.q_opt_rss = ctx->info.q_opt_rss; 8066 vsi->rss_hfunc = hfunc; 8067 netdev_info(vsi->netdev, "Hash function set to: %sToeplitz\n", 8068 hfunc == ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ ? 8069 "Symmetric " : ""); 8070 } 8071 kfree(ctx); 8072 if (err) 8073 return err; 8074 8075 /* Fix the symmetry setting for all existing RSS configurations */ 8076 symm = !!(hfunc == ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ); 8077 return ice_set_rss_cfg_symm(hw, vsi, symm); 8078} 8079 8080/** 8081 * ice_bridge_getlink - Get the hardware bridge mode 8082 * @skb: skb buff 8083 * @pid: process ID 8084 * @seq: RTNL message seq 8085 * @dev: the netdev being configured 8086 * @filter_mask: filter mask passed in 8087 * @nlflags: netlink flags passed in 8088 * 8089 * Return the bridge mode (VEB/VEPA) 8090 */ 8091static int 8092ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 8093 struct net_device *dev, u32 filter_mask, int nlflags) 8094{ 8095 struct ice_pf *pf = ice_netdev_to_pf(dev); 8096 u16 bmode; 8097 8098 bmode = pf->first_sw->bridge_mode; 8099 8100 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 8101 filter_mask, NULL); 8102} 8103 8104/** 8105 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 8106 * @vsi: Pointer to VSI structure 8107 * @bmode: Hardware bridge mode (VEB/VEPA) 8108 * 8109 * Returns 0 on success, negative on failure 8110 */ 8111static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 8112{ 8113 struct ice_aqc_vsi_props *vsi_props; 8114 struct ice_hw *hw = &vsi->back->hw; 8115 struct ice_vsi_ctx *ctxt; 8116 int ret; 8117 8118 vsi_props = &vsi->info; 8119 8120 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 8121 if (!ctxt) 8122 return -ENOMEM; 8123 8124 ctxt->info = vsi->info; 8125 8126 if (bmode == BRIDGE_MODE_VEB) 8127 /* change from VEPA to VEB mode */ 8128 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 8129 else 8130 /* change from VEB to VEPA mode */ 8131 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 8132 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 8133 8134 ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 8135 if (ret) { 8136 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n", 8137 bmode, ret, libie_aq_str(hw->adminq.sq_last_status)); 8138 goto out; 8139 } 8140 /* Update sw flags for book keeping */ 8141 vsi_props->sw_flags = ctxt->info.sw_flags; 8142 8143out: 8144 kfree(ctxt); 8145 return ret; 8146} 8147 8148/** 8149 * ice_bridge_setlink - Set the hardware bridge mode 8150 * @dev: the netdev being configured 8151 * @nlh: RTNL message 8152 * @flags: bridge setlink flags 8153 * @extack: netlink extended ack 8154 * 8155 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 8156 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 8157 * not already set for all VSIs connected to this switch. And also update the 8158 * unicast switch filter rules for the corresponding switch of the netdev. 8159 */ 8160static int 8161ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 8162 u16 __always_unused flags, 8163 struct netlink_ext_ack __always_unused *extack) 8164{ 8165 struct ice_pf *pf = ice_netdev_to_pf(dev); 8166 struct nlattr *attr, *br_spec; 8167 struct ice_hw *hw = &pf->hw; 8168 struct ice_sw *pf_sw; 8169 int rem, v, err = 0; 8170 8171 pf_sw = pf->first_sw; 8172 /* find the attribute in the netlink message */ 8173 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 8174 if (!br_spec) 8175 return -EINVAL; 8176 8177 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 8178 __u16 mode = nla_get_u16(attr); 8179 8180 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 8181 return -EINVAL; 8182 /* Continue if bridge mode is not being flipped */ 8183 if (mode == pf_sw->bridge_mode) 8184 continue; 8185 /* Iterates through the PF VSI list and update the loopback 8186 * mode of the VSI 8187 */ 8188 ice_for_each_vsi(pf, v) { 8189 if (!pf->vsi[v]) 8190 continue; 8191 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 8192 if (err) 8193 return err; 8194 } 8195 8196 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 8197 /* Update the unicast switch filter rules for the corresponding 8198 * switch of the netdev 8199 */ 8200 err = ice_update_sw_rule_bridge_mode(hw); 8201 if (err) { 8202 netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n", 8203 mode, err, 8204 libie_aq_str(hw->adminq.sq_last_status)); 8205 /* revert hw->evb_veb */ 8206 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 8207 return err; 8208 } 8209 8210 pf_sw->bridge_mode = mode; 8211 } 8212 8213 return 0; 8214} 8215 8216/** 8217 * ice_tx_timeout - Respond to a Tx Hang 8218 * @netdev: network interface device structure 8219 * @txqueue: Tx queue 8220 */ 8221void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 8222{ 8223 struct ice_netdev_priv *np = netdev_priv(netdev); 8224 struct ice_tx_ring *tx_ring = NULL; 8225 struct ice_vsi *vsi = np->vsi; 8226 struct ice_pf *pf = vsi->back; 8227 u32 i; 8228 8229 pf->tx_timeout_count++; 8230 8231 /* Check if PFC is enabled for the TC to which the queue belongs 8232 * to. If yes then Tx timeout is not caused by a hung queue, no 8233 * need to reset and rebuild 8234 */ 8235 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 8236 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 8237 txqueue); 8238 return; 8239 } 8240 8241 /* now that we have an index, find the tx_ring struct */ 8242 ice_for_each_txq(vsi, i) 8243 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 8244 if (txqueue == vsi->tx_rings[i]->q_index) { 8245 tx_ring = vsi->tx_rings[i]; 8246 break; 8247 } 8248 8249 /* Reset recovery level if enough time has elapsed after last timeout. 8250 * Also ensure no new reset action happens before next timeout period. 8251 */ 8252 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 8253 pf->tx_timeout_recovery_level = 1; 8254 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 8255 netdev->watchdog_timeo))) 8256 return; 8257 8258 if (tx_ring) { 8259 struct ice_hw *hw = &pf->hw; 8260 u32 head, intr = 0; 8261 8262 head = FIELD_GET(QTX_COMM_HEAD_HEAD_M, 8263 rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue]))); 8264 /* Read interrupt register */ 8265 intr = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 8266 8267 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 8268 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 8269 head, tx_ring->next_to_use, intr); 8270 8271 ice_prep_tx_hang_report(pf, tx_ring, vsi->vsi_num, head, intr); 8272 } 8273 8274 pf->tx_timeout_last_recovery = jiffies; 8275 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 8276 pf->tx_timeout_recovery_level, txqueue); 8277 8278 switch (pf->tx_timeout_recovery_level) { 8279 case 1: 8280 set_bit(ICE_PFR_REQ, pf->state); 8281 break; 8282 case 2: 8283 set_bit(ICE_CORER_REQ, pf->state); 8284 break; 8285 case 3: 8286 set_bit(ICE_GLOBR_REQ, pf->state); 8287 break; 8288 default: 8289 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 8290 set_bit(ICE_DOWN, pf->state); 8291 set_bit(ICE_VSI_NEEDS_RESTART, vsi->state); 8292 set_bit(ICE_SERVICE_DIS, pf->state); 8293 break; 8294 } 8295 8296 ice_service_task_schedule(pf); 8297 pf->tx_timeout_recovery_level++; 8298} 8299 8300/** 8301 * ice_setup_tc_cls_flower - flower classifier offloads 8302 * @np: net device to configure 8303 * @filter_dev: device on which filter is added 8304 * @cls_flower: offload data 8305 * @ingress: if the rule is added to an ingress block 8306 * 8307 * Return: 0 if the flower was successfully added or deleted, 8308 * negative error code otherwise. 8309 */ 8310static int 8311ice_setup_tc_cls_flower(struct ice_netdev_priv *np, 8312 struct net_device *filter_dev, 8313 struct flow_cls_offload *cls_flower, 8314 bool ingress) 8315{ 8316 struct ice_vsi *vsi = np->vsi; 8317 8318 if (cls_flower->common.chain_index) 8319 return -EOPNOTSUPP; 8320 8321 switch (cls_flower->command) { 8322 case FLOW_CLS_REPLACE: 8323 return ice_add_cls_flower(filter_dev, vsi, cls_flower, ingress); 8324 case FLOW_CLS_DESTROY: 8325 return ice_del_cls_flower(vsi, cls_flower); 8326 default: 8327 return -EINVAL; 8328 } 8329} 8330 8331/** 8332 * ice_setup_tc_block_cb_ingress - callback handler for ingress TC block 8333 * @type: TC SETUP type 8334 * @type_data: TC flower offload data that contains user input 8335 * @cb_priv: netdev private data 8336 * 8337 * Return: 0 if the setup was successful, negative error code otherwise. 8338 */ 8339static int 8340ice_setup_tc_block_cb_ingress(enum tc_setup_type type, void *type_data, 8341 void *cb_priv) 8342{ 8343 struct ice_netdev_priv *np = cb_priv; 8344 8345 switch (type) { 8346 case TC_SETUP_CLSFLOWER: 8347 return ice_setup_tc_cls_flower(np, np->vsi->netdev, 8348 type_data, true); 8349 default: 8350 return -EOPNOTSUPP; 8351 } 8352} 8353 8354/** 8355 * ice_setup_tc_block_cb_egress - callback handler for egress TC block 8356 * @type: TC SETUP type 8357 * @type_data: TC flower offload data that contains user input 8358 * @cb_priv: netdev private data 8359 * 8360 * Return: 0 if the setup was successful, negative error code otherwise. 8361 */ 8362static int 8363ice_setup_tc_block_cb_egress(enum tc_setup_type type, void *type_data, 8364 void *cb_priv) 8365{ 8366 struct ice_netdev_priv *np = cb_priv; 8367 8368 switch (type) { 8369 case TC_SETUP_CLSFLOWER: 8370 return ice_setup_tc_cls_flower(np, np->vsi->netdev, 8371 type_data, false); 8372 default: 8373 return -EOPNOTSUPP; 8374 } 8375} 8376 8377/** 8378 * ice_validate_mqprio_qopt - Validate TCF input parameters 8379 * @vsi: Pointer to VSI 8380 * @mqprio_qopt: input parameters for mqprio queue configuration 8381 * 8382 * This function validates MQPRIO params, such as qcount (power of 2 wherever 8383 * needed), and make sure user doesn't specify qcount and BW rate limit 8384 * for TCs, which are more than "num_tc" 8385 */ 8386static int 8387ice_validate_mqprio_qopt(struct ice_vsi *vsi, 8388 struct tc_mqprio_qopt_offload *mqprio_qopt) 8389{ 8390 int non_power_of_2_qcount = 0; 8391 struct ice_pf *pf = vsi->back; 8392 int max_rss_q_cnt = 0; 8393 u64 sum_min_rate = 0; 8394 struct device *dev; 8395 int i, speed; 8396 u8 num_tc; 8397 8398 if (vsi->type != ICE_VSI_PF) 8399 return -EINVAL; 8400 8401 if (mqprio_qopt->qopt.offset[0] != 0 || 8402 mqprio_qopt->qopt.num_tc < 1 || 8403 mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC) 8404 return -EINVAL; 8405 8406 dev = ice_pf_to_dev(pf); 8407 vsi->ch_rss_size = 0; 8408 num_tc = mqprio_qopt->qopt.num_tc; 8409 speed = ice_get_link_speed_kbps(vsi); 8410 8411 for (i = 0; num_tc; i++) { 8412 int qcount = mqprio_qopt->qopt.count[i]; 8413 u64 max_rate, min_rate, rem; 8414 8415 if (!qcount) 8416 return -EINVAL; 8417 8418 if (is_power_of_2(qcount)) { 8419 if (non_power_of_2_qcount && 8420 qcount > non_power_of_2_qcount) { 8421 dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n", 8422 qcount, non_power_of_2_qcount); 8423 return -EINVAL; 8424 } 8425 if (qcount > max_rss_q_cnt) 8426 max_rss_q_cnt = qcount; 8427 } else { 8428 if (non_power_of_2_qcount && 8429 qcount != non_power_of_2_qcount) { 8430 dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n", 8431 qcount, non_power_of_2_qcount); 8432 return -EINVAL; 8433 } 8434 if (qcount < max_rss_q_cnt) { 8435 dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n", 8436 qcount, max_rss_q_cnt); 8437 return -EINVAL; 8438 } 8439 max_rss_q_cnt = qcount; 8440 non_power_of_2_qcount = qcount; 8441 } 8442 8443 /* TC command takes input in K/N/Gbps or K/M/Gbit etc but 8444 * converts the bandwidth rate limit into Bytes/s when 8445 * passing it down to the driver. So convert input bandwidth 8446 * from Bytes/s to Kbps 8447 */ 8448 max_rate = mqprio_qopt->max_rate[i]; 8449 max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR); 8450 8451 /* min_rate is minimum guaranteed rate and it can't be zero */ 8452 min_rate = mqprio_qopt->min_rate[i]; 8453 min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR); 8454 sum_min_rate += min_rate; 8455 8456 if (min_rate && min_rate < ICE_MIN_BW_LIMIT) { 8457 dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i, 8458 min_rate, ICE_MIN_BW_LIMIT); 8459 return -EINVAL; 8460 } 8461 8462 if (max_rate && max_rate > speed) { 8463 dev_err(dev, "TC%d: max_rate(%llu Kbps) > link speed of %u Kbps\n", 8464 i, max_rate, speed); 8465 return -EINVAL; 8466 } 8467 8468 iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem); 8469 if (rem) { 8470 dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps", 8471 i, ICE_MIN_BW_LIMIT); 8472 return -EINVAL; 8473 } 8474 8475 iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem); 8476 if (rem) { 8477 dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps", 8478 i, ICE_MIN_BW_LIMIT); 8479 return -EINVAL; 8480 } 8481 8482 /* min_rate can't be more than max_rate, except when max_rate 8483 * is zero (implies max_rate sought is max line rate). In such 8484 * a case min_rate can be more than max. 8485 */ 8486 if (max_rate && min_rate > max_rate) { 8487 dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n", 8488 min_rate, max_rate); 8489 return -EINVAL; 8490 } 8491 8492 if (i >= mqprio_qopt->qopt.num_tc - 1) 8493 break; 8494 if (mqprio_qopt->qopt.offset[i + 1] != 8495 (mqprio_qopt->qopt.offset[i] + qcount)) 8496 return -EINVAL; 8497 } 8498 if (vsi->num_rxq < 8499 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 8500 return -EINVAL; 8501 if (vsi->num_txq < 8502 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 8503 return -EINVAL; 8504 8505 if (sum_min_rate && sum_min_rate > (u64)speed) { 8506 dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n", 8507 sum_min_rate, speed); 8508 return -EINVAL; 8509 } 8510 8511 /* make sure vsi->ch_rss_size is set correctly based on TC's qcount */ 8512 vsi->ch_rss_size = max_rss_q_cnt; 8513 8514 return 0; 8515} 8516 8517/** 8518 * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF 8519 * @pf: ptr to PF device 8520 * @vsi: ptr to VSI 8521 */ 8522static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi) 8523{ 8524 struct device *dev = ice_pf_to_dev(pf); 8525 bool added = false; 8526 struct ice_hw *hw; 8527 int flow; 8528 8529 if (!(vsi->num_gfltr || vsi->num_bfltr)) 8530 return -EINVAL; 8531 8532 hw = &pf->hw; 8533 for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) { 8534 struct ice_fd_hw_prof *prof; 8535 int tun, status; 8536 u64 entry_h; 8537 8538 if (!(hw->fdir_prof && hw->fdir_prof[flow] && 8539 hw->fdir_prof[flow]->cnt)) 8540 continue; 8541 8542 for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { 8543 enum ice_flow_priority prio; 8544 8545 /* add this VSI to FDir profile for this flow */ 8546 prio = ICE_FLOW_PRIO_NORMAL; 8547 prof = hw->fdir_prof[flow]; 8548 status = ice_flow_add_entry(hw, ICE_BLK_FD, 8549 prof->prof_id[tun], 8550 prof->vsi_h[0], vsi->idx, 8551 prio, prof->fdir_seg[tun], 8552 &entry_h); 8553 if (status) { 8554 dev_err(dev, "channel VSI idx %d, not able to add to group %d\n", 8555 vsi->idx, flow); 8556 continue; 8557 } 8558 8559 prof->entry_h[prof->cnt][tun] = entry_h; 8560 } 8561 8562 /* store VSI for filter replay and delete */ 8563 prof->vsi_h[prof->cnt] = vsi->idx; 8564 prof->cnt++; 8565 8566 added = true; 8567 dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx, 8568 flow); 8569 } 8570 8571 if (!added) 8572 dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx); 8573 8574 return 0; 8575} 8576 8577/** 8578 * ice_add_channel - add a channel by adding VSI 8579 * @pf: ptr to PF device 8580 * @sw_id: underlying HW switching element ID 8581 * @ch: ptr to channel structure 8582 * 8583 * Add a channel (VSI) using add_vsi and queue_map 8584 */ 8585static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch) 8586{ 8587 struct device *dev = ice_pf_to_dev(pf); 8588 struct ice_vsi *vsi; 8589 8590 if (ch->type != ICE_VSI_CHNL) { 8591 dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type); 8592 return -EINVAL; 8593 } 8594 8595 vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch); 8596 if (!vsi || vsi->type != ICE_VSI_CHNL) { 8597 dev_err(dev, "create chnl VSI failure\n"); 8598 return -EINVAL; 8599 } 8600 8601 ice_add_vsi_to_fdir(pf, vsi); 8602 8603 ch->sw_id = sw_id; 8604 ch->vsi_num = vsi->vsi_num; 8605 ch->info.mapping_flags = vsi->info.mapping_flags; 8606 ch->ch_vsi = vsi; 8607 /* set the back pointer of channel for newly created VSI */ 8608 vsi->ch = ch; 8609 8610 memcpy(&ch->info.q_mapping, &vsi->info.q_mapping, 8611 sizeof(vsi->info.q_mapping)); 8612 memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping, 8613 sizeof(vsi->info.tc_mapping)); 8614 8615 return 0; 8616} 8617 8618/** 8619 * ice_chnl_cfg_res 8620 * @vsi: the VSI being setup 8621 * @ch: ptr to channel structure 8622 * 8623 * Configure channel specific resources such as rings, vector. 8624 */ 8625static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch) 8626{ 8627 int i; 8628 8629 for (i = 0; i < ch->num_txq; i++) { 8630 struct ice_q_vector *tx_q_vector, *rx_q_vector; 8631 struct ice_ring_container *rc; 8632 struct ice_tx_ring *tx_ring; 8633 struct ice_rx_ring *rx_ring; 8634 8635 tx_ring = vsi->tx_rings[ch->base_q + i]; 8636 rx_ring = vsi->rx_rings[ch->base_q + i]; 8637 if (!tx_ring || !rx_ring) 8638 continue; 8639 8640 /* setup ring being channel enabled */ 8641 tx_ring->ch = ch; 8642 rx_ring->ch = ch; 8643 8644 /* following code block sets up vector specific attributes */ 8645 tx_q_vector = tx_ring->q_vector; 8646 rx_q_vector = rx_ring->q_vector; 8647 if (!tx_q_vector && !rx_q_vector) 8648 continue; 8649 8650 if (tx_q_vector) { 8651 tx_q_vector->ch = ch; 8652 /* setup Tx and Rx ITR setting if DIM is off */ 8653 rc = &tx_q_vector->tx; 8654 if (!ITR_IS_DYNAMIC(rc)) 8655 ice_write_itr(rc, rc->itr_setting); 8656 } 8657 if (rx_q_vector) { 8658 rx_q_vector->ch = ch; 8659 /* setup Tx and Rx ITR setting if DIM is off */ 8660 rc = &rx_q_vector->rx; 8661 if (!ITR_IS_DYNAMIC(rc)) 8662 ice_write_itr(rc, rc->itr_setting); 8663 } 8664 } 8665 8666 /* it is safe to assume that, if channel has non-zero num_t[r]xq, then 8667 * GLINT_ITR register would have written to perform in-context 8668 * update, hence perform flush 8669 */ 8670 if (ch->num_txq || ch->num_rxq) 8671 ice_flush(&vsi->back->hw); 8672} 8673 8674/** 8675 * ice_cfg_chnl_all_res - configure channel resources 8676 * @vsi: pte to main_vsi 8677 * @ch: ptr to channel structure 8678 * 8679 * This function configures channel specific resources such as flow-director 8680 * counter index, and other resources such as queues, vectors, ITR settings 8681 */ 8682static void 8683ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch) 8684{ 8685 /* configure channel (aka ADQ) resources such as queues, vectors, 8686 * ITR settings for channel specific vectors and anything else 8687 */ 8688 ice_chnl_cfg_res(vsi, ch); 8689} 8690 8691/** 8692 * ice_setup_hw_channel - setup new channel 8693 * @pf: ptr to PF device 8694 * @vsi: the VSI being setup 8695 * @ch: ptr to channel structure 8696 * @sw_id: underlying HW switching element ID 8697 * @type: type of channel to be created (VMDq2/VF) 8698 * 8699 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8700 * and configures Tx rings accordingly 8701 */ 8702static int 8703ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8704 struct ice_channel *ch, u16 sw_id, u8 type) 8705{ 8706 struct device *dev = ice_pf_to_dev(pf); 8707 int ret; 8708 8709 ch->base_q = vsi->next_base_q; 8710 ch->type = type; 8711 8712 ret = ice_add_channel(pf, sw_id, ch); 8713 if (ret) { 8714 dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id); 8715 return ret; 8716 } 8717 8718 /* configure/setup ADQ specific resources */ 8719 ice_cfg_chnl_all_res(vsi, ch); 8720 8721 /* make sure to update the next_base_q so that subsequent channel's 8722 * (aka ADQ) VSI queue map is correct 8723 */ 8724 vsi->next_base_q = vsi->next_base_q + ch->num_rxq; 8725 dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num, 8726 ch->num_rxq); 8727 8728 return 0; 8729} 8730 8731/** 8732 * ice_setup_channel - setup new channel using uplink element 8733 * @pf: ptr to PF device 8734 * @vsi: the VSI being setup 8735 * @ch: ptr to channel structure 8736 * 8737 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8738 * and uplink switching element 8739 */ 8740static bool 8741ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8742 struct ice_channel *ch) 8743{ 8744 struct device *dev = ice_pf_to_dev(pf); 8745 u16 sw_id; 8746 int ret; 8747 8748 if (vsi->type != ICE_VSI_PF) { 8749 dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type); 8750 return false; 8751 } 8752 8753 sw_id = pf->first_sw->sw_id; 8754 8755 /* create channel (VSI) */ 8756 ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL); 8757 if (ret) { 8758 dev_err(dev, "failed to setup hw_channel\n"); 8759 return false; 8760 } 8761 dev_dbg(dev, "successfully created channel()\n"); 8762 8763 return ch->ch_vsi ? true : false; 8764} 8765 8766/** 8767 * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 8768 * @vsi: VSI to be configured 8769 * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit 8770 * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit 8771 */ 8772static int 8773ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate) 8774{ 8775 int err; 8776 8777 err = ice_set_min_bw_limit(vsi, min_tx_rate); 8778 if (err) 8779 return err; 8780 8781 return ice_set_max_bw_limit(vsi, max_tx_rate); 8782} 8783 8784/** 8785 * ice_create_q_channel - function to create channel 8786 * @vsi: VSI to be configured 8787 * @ch: ptr to channel (it contains channel specific params) 8788 * 8789 * This function creates channel (VSI) using num_queues specified by user, 8790 * reconfigs RSS if needed. 8791 */ 8792static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch) 8793{ 8794 struct ice_pf *pf = vsi->back; 8795 struct device *dev; 8796 8797 if (!ch) 8798 return -EINVAL; 8799 8800 dev = ice_pf_to_dev(pf); 8801 if (!ch->num_txq || !ch->num_rxq) { 8802 dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq); 8803 return -EINVAL; 8804 } 8805 8806 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) { 8807 dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n", 8808 vsi->cnt_q_avail, ch->num_txq); 8809 return -EINVAL; 8810 } 8811 8812 if (!ice_setup_channel(pf, vsi, ch)) { 8813 dev_info(dev, "Failed to setup channel\n"); 8814 return -EINVAL; 8815 } 8816 /* configure BW rate limit */ 8817 if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) { 8818 int ret; 8819 8820 ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate, 8821 ch->min_tx_rate); 8822 if (ret) 8823 dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n", 8824 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8825 else 8826 dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n", 8827 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8828 } 8829 8830 vsi->cnt_q_avail -= ch->num_txq; 8831 8832 return 0; 8833} 8834 8835/** 8836 * ice_rem_all_chnl_fltrs - removes all channel filters 8837 * @pf: ptr to PF, TC-flower based filter are tracked at PF level 8838 * 8839 * Remove all advanced switch filters only if they are channel specific 8840 * tc-flower based filter 8841 */ 8842static void ice_rem_all_chnl_fltrs(struct ice_pf *pf) 8843{ 8844 struct ice_tc_flower_fltr *fltr; 8845 struct hlist_node *node; 8846 8847 /* to remove all channel filters, iterate an ordered list of filters */ 8848 hlist_for_each_entry_safe(fltr, node, 8849 &pf->tc_flower_fltr_list, 8850 tc_flower_node) { 8851 struct ice_rule_query_data rule; 8852 int status; 8853 8854 /* for now process only channel specific filters */ 8855 if (!ice_is_chnl_fltr(fltr)) 8856 continue; 8857 8858 rule.rid = fltr->rid; 8859 rule.rule_id = fltr->rule_id; 8860 rule.vsi_handle = fltr->dest_vsi_handle; 8861 status = ice_rem_adv_rule_by_id(&pf->hw, &rule); 8862 if (status) { 8863 if (status == -ENOENT) 8864 dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n", 8865 rule.rule_id); 8866 else 8867 dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n", 8868 status); 8869 } else if (fltr->dest_vsi) { 8870 /* update advanced switch filter count */ 8871 if (fltr->dest_vsi->type == ICE_VSI_CHNL) { 8872 u32 flags = fltr->flags; 8873 8874 fltr->dest_vsi->num_chnl_fltr--; 8875 if (flags & (ICE_TC_FLWR_FIELD_DST_MAC | 8876 ICE_TC_FLWR_FIELD_ENC_DST_MAC)) 8877 pf->num_dmac_chnl_fltrs--; 8878 } 8879 } 8880 8881 hlist_del(&fltr->tc_flower_node); 8882 kfree(fltr); 8883 } 8884} 8885 8886/** 8887 * ice_remove_q_channels - Remove queue channels for the TCs 8888 * @vsi: VSI to be configured 8889 * @rem_fltr: delete advanced switch filter or not 8890 * 8891 * Remove queue channels for the TCs 8892 */ 8893static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr) 8894{ 8895 struct ice_channel *ch, *ch_tmp; 8896 struct ice_pf *pf = vsi->back; 8897 int i; 8898 8899 /* remove all tc-flower based filter if they are channel filters only */ 8900 if (rem_fltr) 8901 ice_rem_all_chnl_fltrs(pf); 8902 8903 /* remove ntuple filters since queue configuration is being changed */ 8904 if (vsi->netdev->features & NETIF_F_NTUPLE) { 8905 struct ice_hw *hw = &pf->hw; 8906 8907 mutex_lock(&hw->fdir_fltr_lock); 8908 ice_fdir_del_all_fltrs(vsi); 8909 mutex_unlock(&hw->fdir_fltr_lock); 8910 } 8911 8912 /* perform cleanup for channels if they exist */ 8913 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 8914 struct ice_vsi *ch_vsi; 8915 8916 list_del(&ch->list); 8917 ch_vsi = ch->ch_vsi; 8918 if (!ch_vsi) { 8919 kfree(ch); 8920 continue; 8921 } 8922 8923 /* Reset queue contexts */ 8924 for (i = 0; i < ch->num_rxq; i++) { 8925 struct ice_tx_ring *tx_ring; 8926 struct ice_rx_ring *rx_ring; 8927 8928 tx_ring = vsi->tx_rings[ch->base_q + i]; 8929 rx_ring = vsi->rx_rings[ch->base_q + i]; 8930 if (tx_ring) { 8931 tx_ring->ch = NULL; 8932 if (tx_ring->q_vector) 8933 tx_ring->q_vector->ch = NULL; 8934 } 8935 if (rx_ring) { 8936 rx_ring->ch = NULL; 8937 if (rx_ring->q_vector) 8938 rx_ring->q_vector->ch = NULL; 8939 } 8940 } 8941 8942 /* Release FD resources for the channel VSI */ 8943 ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx); 8944 8945 /* clear the VSI from scheduler tree */ 8946 ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx); 8947 8948 /* Delete VSI from FW, PF and HW VSI arrays */ 8949 ice_vsi_delete(ch->ch_vsi); 8950 8951 /* free the channel */ 8952 kfree(ch); 8953 } 8954 8955 /* clear the channel VSI map which is stored in main VSI */ 8956 ice_for_each_chnl_tc(i) 8957 vsi->tc_map_vsi[i] = NULL; 8958 8959 /* reset main VSI's all TC information */ 8960 vsi->all_enatc = 0; 8961 vsi->all_numtc = 0; 8962} 8963 8964/** 8965 * ice_rebuild_channels - rebuild channel 8966 * @pf: ptr to PF 8967 * 8968 * Recreate channel VSIs and replay filters 8969 */ 8970static int ice_rebuild_channels(struct ice_pf *pf) 8971{ 8972 struct device *dev = ice_pf_to_dev(pf); 8973 struct ice_vsi *main_vsi; 8974 bool rem_adv_fltr = true; 8975 struct ice_channel *ch; 8976 struct ice_vsi *vsi; 8977 int tc_idx = 1; 8978 int i, err; 8979 8980 main_vsi = ice_get_main_vsi(pf); 8981 if (!main_vsi) 8982 return 0; 8983 8984 if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) || 8985 main_vsi->old_numtc == 1) 8986 return 0; /* nothing to be done */ 8987 8988 /* reconfigure main VSI based on old value of TC and cached values 8989 * for MQPRIO opts 8990 */ 8991 err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc); 8992 if (err) { 8993 dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n", 8994 main_vsi->old_ena_tc, main_vsi->vsi_num); 8995 return err; 8996 } 8997 8998 /* rebuild ADQ VSIs */ 8999 ice_for_each_vsi(pf, i) { 9000 enum ice_vsi_type type; 9001 9002 vsi = pf->vsi[i]; 9003 if (!vsi || vsi->type != ICE_VSI_CHNL) 9004 continue; 9005 9006 type = vsi->type; 9007 9008 /* rebuild ADQ VSI */ 9009 err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT); 9010 if (err) { 9011 dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n", 9012 ice_vsi_type_str(type), vsi->idx, err); 9013 goto cleanup; 9014 } 9015 9016 /* Re-map HW VSI number, using VSI handle that has been 9017 * previously validated in ice_replay_vsi() call above 9018 */ 9019 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 9020 9021 /* replay filters for the VSI */ 9022 err = ice_replay_vsi(&pf->hw, vsi->idx); 9023 if (err) { 9024 dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n", 9025 ice_vsi_type_str(type), err, vsi->idx); 9026 rem_adv_fltr = false; 9027 goto cleanup; 9028 } 9029 dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n", 9030 ice_vsi_type_str(type), vsi->idx); 9031 9032 /* store ADQ VSI at correct TC index in main VSI's 9033 * map of TC to VSI 9034 */ 9035 main_vsi->tc_map_vsi[tc_idx++] = vsi; 9036 } 9037 9038 /* ADQ VSI(s) has been rebuilt successfully, so setup 9039 * channel for main VSI's Tx and Rx rings 9040 */ 9041 list_for_each_entry(ch, &main_vsi->ch_list, list) { 9042 struct ice_vsi *ch_vsi; 9043 9044 ch_vsi = ch->ch_vsi; 9045 if (!ch_vsi) 9046 continue; 9047 9048 /* reconfig channel resources */ 9049 ice_cfg_chnl_all_res(main_vsi, ch); 9050 9051 /* replay BW rate limit if it is non-zero */ 9052 if (!ch->max_tx_rate && !ch->min_tx_rate) 9053 continue; 9054 9055 err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate, 9056 ch->min_tx_rate); 9057 if (err) 9058 dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 9059 err, ch->max_tx_rate, ch->min_tx_rate, 9060 ch_vsi->vsi_num); 9061 else 9062 dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 9063 ch->max_tx_rate, ch->min_tx_rate, 9064 ch_vsi->vsi_num); 9065 } 9066 9067 /* reconfig RSS for main VSI */ 9068 if (main_vsi->ch_rss_size) 9069 ice_vsi_cfg_rss_lut_key(main_vsi); 9070 9071 return 0; 9072 9073cleanup: 9074 ice_remove_q_channels(main_vsi, rem_adv_fltr); 9075 return err; 9076} 9077 9078/** 9079 * ice_create_q_channels - Add queue channel for the given TCs 9080 * @vsi: VSI to be configured 9081 * 9082 * Configures queue channel mapping to the given TCs 9083 */ 9084static int ice_create_q_channels(struct ice_vsi *vsi) 9085{ 9086 struct ice_pf *pf = vsi->back; 9087 struct ice_channel *ch; 9088 int ret = 0, i; 9089 9090 ice_for_each_chnl_tc(i) { 9091 if (!(vsi->all_enatc & BIT(i))) 9092 continue; 9093 9094 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 9095 if (!ch) { 9096 ret = -ENOMEM; 9097 goto err_free; 9098 } 9099 INIT_LIST_HEAD(&ch->list); 9100 ch->num_rxq = vsi->mqprio_qopt.qopt.count[i]; 9101 ch->num_txq = vsi->mqprio_qopt.qopt.count[i]; 9102 ch->base_q = vsi->mqprio_qopt.qopt.offset[i]; 9103 ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i]; 9104 ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i]; 9105 9106 /* convert to Kbits/s */ 9107 if (ch->max_tx_rate) 9108 ch->max_tx_rate = div_u64(ch->max_tx_rate, 9109 ICE_BW_KBPS_DIVISOR); 9110 if (ch->min_tx_rate) 9111 ch->min_tx_rate = div_u64(ch->min_tx_rate, 9112 ICE_BW_KBPS_DIVISOR); 9113 9114 ret = ice_create_q_channel(vsi, ch); 9115 if (ret) { 9116 dev_err(ice_pf_to_dev(pf), 9117 "failed creating channel TC:%d\n", i); 9118 kfree(ch); 9119 goto err_free; 9120 } 9121 list_add_tail(&ch->list, &vsi->ch_list); 9122 vsi->tc_map_vsi[i] = ch->ch_vsi; 9123 dev_dbg(ice_pf_to_dev(pf), 9124 "successfully created channel: VSI %p\n", ch->ch_vsi); 9125 } 9126 return 0; 9127 9128err_free: 9129 ice_remove_q_channels(vsi, false); 9130 9131 return ret; 9132} 9133 9134/** 9135 * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes 9136 * @netdev: net device to configure 9137 * @type_data: TC offload data 9138 */ 9139static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data) 9140{ 9141 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 9142 struct ice_netdev_priv *np = netdev_priv(netdev); 9143 struct ice_vsi *vsi = np->vsi; 9144 struct ice_pf *pf = vsi->back; 9145 u16 mode, ena_tc_qdisc = 0; 9146 int cur_txq, cur_rxq; 9147 u8 hw = 0, num_tcf; 9148 struct device *dev; 9149 int ret, i; 9150 9151 dev = ice_pf_to_dev(pf); 9152 num_tcf = mqprio_qopt->qopt.num_tc; 9153 hw = mqprio_qopt->qopt.hw; 9154 mode = mqprio_qopt->mode; 9155 if (!hw) { 9156 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 9157 vsi->ch_rss_size = 0; 9158 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 9159 goto config_tcf; 9160 } 9161 9162 /* Generate queue region map for number of TCF requested */ 9163 for (i = 0; i < num_tcf; i++) 9164 ena_tc_qdisc |= BIT(i); 9165 9166 switch (mode) { 9167 case TC_MQPRIO_MODE_CHANNEL: 9168 9169 if (pf->hw.port_info->is_custom_tx_enabled) { 9170 dev_err(dev, "Custom Tx scheduler feature enabled, can't configure ADQ\n"); 9171 return -EBUSY; 9172 } 9173 ice_tear_down_devlink_rate_tree(pf); 9174 9175 ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt); 9176 if (ret) { 9177 netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n", 9178 ret); 9179 return ret; 9180 } 9181 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 9182 set_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 9183 /* don't assume state of hw_tc_offload during driver load 9184 * and set the flag for TC flower filter if hw_tc_offload 9185 * already ON 9186 */ 9187 if (vsi->netdev->features & NETIF_F_HW_TC) 9188 set_bit(ICE_FLAG_CLS_FLOWER, pf->flags); 9189 break; 9190 default: 9191 return -EINVAL; 9192 } 9193 9194config_tcf: 9195 9196 /* Requesting same TCF configuration as already enabled */ 9197 if (ena_tc_qdisc == vsi->tc_cfg.ena_tc && 9198 mode != TC_MQPRIO_MODE_CHANNEL) 9199 return 0; 9200 9201 /* Pause VSI queues */ 9202 ice_dis_vsi(vsi, true); 9203 9204 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) 9205 ice_remove_q_channels(vsi, true); 9206 9207 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 9208 vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf), 9209 num_online_cpus()); 9210 vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf), 9211 num_online_cpus()); 9212 } else { 9213 /* logic to rebuild VSI, same like ethtool -L */ 9214 u16 offset = 0, qcount_tx = 0, qcount_rx = 0; 9215 9216 for (i = 0; i < num_tcf; i++) { 9217 if (!(ena_tc_qdisc & BIT(i))) 9218 continue; 9219 9220 offset = vsi->mqprio_qopt.qopt.offset[i]; 9221 qcount_rx = vsi->mqprio_qopt.qopt.count[i]; 9222 qcount_tx = vsi->mqprio_qopt.qopt.count[i]; 9223 } 9224 vsi->req_txq = offset + qcount_tx; 9225 vsi->req_rxq = offset + qcount_rx; 9226 9227 /* store away original rss_size info, so that it gets reused 9228 * form ice_vsi_rebuild during tc-qdisc delete stage - to 9229 * determine, what should be the rss_sizefor main VSI 9230 */ 9231 vsi->orig_rss_size = vsi->rss_size; 9232 } 9233 9234 /* save current values of Tx and Rx queues before calling VSI rebuild 9235 * for fallback option 9236 */ 9237 cur_txq = vsi->num_txq; 9238 cur_rxq = vsi->num_rxq; 9239 9240 /* proceed with rebuild main VSI using correct number of queues */ 9241 ret = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT); 9242 if (ret) { 9243 /* fallback to current number of queues */ 9244 dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n"); 9245 vsi->req_txq = cur_txq; 9246 vsi->req_rxq = cur_rxq; 9247 clear_bit(ICE_RESET_FAILED, pf->state); 9248 if (ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT)) { 9249 dev_err(dev, "Rebuild of main VSI failed again\n"); 9250 return ret; 9251 } 9252 } 9253 9254 vsi->all_numtc = num_tcf; 9255 vsi->all_enatc = ena_tc_qdisc; 9256 ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc); 9257 if (ret) { 9258 netdev_err(netdev, "failed configuring TC for VSI id=%d\n", 9259 vsi->vsi_num); 9260 goto exit; 9261 } 9262 9263 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 9264 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 9265 u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0]; 9266 9267 /* set TC0 rate limit if specified */ 9268 if (max_tx_rate || min_tx_rate) { 9269 /* convert to Kbits/s */ 9270 if (max_tx_rate) 9271 max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR); 9272 if (min_tx_rate) 9273 min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR); 9274 9275 ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate); 9276 if (!ret) { 9277 dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n", 9278 max_tx_rate, min_tx_rate, vsi->vsi_num); 9279 } else { 9280 dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n", 9281 max_tx_rate, min_tx_rate, vsi->vsi_num); 9282 goto exit; 9283 } 9284 } 9285 ret = ice_create_q_channels(vsi); 9286 if (ret) { 9287 netdev_err(netdev, "failed configuring queue channels\n"); 9288 goto exit; 9289 } else { 9290 netdev_dbg(netdev, "successfully configured channels\n"); 9291 } 9292 } 9293 9294 if (vsi->ch_rss_size) 9295 ice_vsi_cfg_rss_lut_key(vsi); 9296 9297exit: 9298 /* if error, reset the all_numtc and all_enatc */ 9299 if (ret) { 9300 vsi->all_numtc = 0; 9301 vsi->all_enatc = 0; 9302 } 9303 /* resume VSI */ 9304 ice_ena_vsi(vsi, true); 9305 9306 return ret; 9307} 9308 9309/** 9310 * ice_cfg_txtime - configure Tx Time for the Tx ring 9311 * @tx_ring: pointer to the Tx ring structure 9312 * 9313 * Return: 0 on success, negative value on failure. 9314 */ 9315static int ice_cfg_txtime(struct ice_tx_ring *tx_ring) 9316{ 9317 int err, timeout = 50; 9318 struct ice_vsi *vsi; 9319 struct device *dev; 9320 struct ice_pf *pf; 9321 u32 queue; 9322 9323 if (!tx_ring) 9324 return -EINVAL; 9325 9326 vsi = tx_ring->vsi; 9327 pf = vsi->back; 9328 while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) { 9329 timeout--; 9330 if (!timeout) 9331 return -EBUSY; 9332 usleep_range(1000, 2000); 9333 } 9334 9335 queue = tx_ring->q_index; 9336 dev = ice_pf_to_dev(pf); 9337 9338 /* Ignore return value, and always attempt to enable queue. */ 9339 ice_qp_dis(vsi, queue); 9340 9341 err = ice_qp_ena(vsi, queue); 9342 if (err) 9343 dev_err(dev, "Failed to enable Tx queue %d for TxTime configuration\n", 9344 queue); 9345 9346 clear_bit(ICE_CFG_BUSY, pf->state); 9347 return err; 9348} 9349 9350/** 9351 * ice_offload_txtime - set earliest TxTime first 9352 * @netdev: network interface device structure 9353 * @qopt_off: etf queue option offload from the skb to set 9354 * 9355 * Return: 0 on success, negative value on failure. 9356 */ 9357static int ice_offload_txtime(struct net_device *netdev, 9358 void *qopt_off) 9359{ 9360 struct ice_netdev_priv *np = netdev_priv(netdev); 9361 struct ice_pf *pf = np->vsi->back; 9362 struct tc_etf_qopt_offload *qopt; 9363 struct ice_vsi *vsi = np->vsi; 9364 struct ice_tx_ring *tx_ring; 9365 int ret = 0; 9366 9367 if (!ice_is_feature_supported(pf, ICE_F_TXTIME)) 9368 return -EOPNOTSUPP; 9369 9370 qopt = qopt_off; 9371 if (!qopt_off || qopt->queue < 0 || qopt->queue >= vsi->num_txq) 9372 return -EINVAL; 9373 9374 if (qopt->enable) 9375 set_bit(qopt->queue, pf->txtime_txqs); 9376 else 9377 clear_bit(qopt->queue, pf->txtime_txqs); 9378 9379 if (netif_running(vsi->netdev)) { 9380 tx_ring = vsi->tx_rings[qopt->queue]; 9381 ret = ice_cfg_txtime(tx_ring); 9382 if (ret) 9383 goto err; 9384 } 9385 9386 netdev_info(netdev, "%s TxTime on queue: %i\n", 9387 str_enable_disable(qopt->enable), qopt->queue); 9388 return 0; 9389 9390err: 9391 netdev_err(netdev, "Failed to %s TxTime on queue: %i\n", 9392 str_enable_disable(qopt->enable), qopt->queue); 9393 9394 if (qopt->enable) 9395 clear_bit(qopt->queue, pf->txtime_txqs); 9396 return ret; 9397} 9398 9399static LIST_HEAD(ice_block_cb_list); 9400 9401static int 9402ice_setup_tc(struct net_device *netdev, enum tc_setup_type type, 9403 void *type_data) 9404{ 9405 struct ice_netdev_priv *np = netdev_priv(netdev); 9406 enum flow_block_binder_type binder_type; 9407 struct iidc_rdma_core_dev_info *cdev; 9408 struct ice_pf *pf = np->vsi->back; 9409 flow_setup_cb_t *flower_handler; 9410 bool locked = false; 9411 int err; 9412 9413 switch (type) { 9414 case TC_SETUP_BLOCK: 9415 binder_type = 9416 ((struct flow_block_offload *)type_data)->binder_type; 9417 9418 switch (binder_type) { 9419 case FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS: 9420 flower_handler = ice_setup_tc_block_cb_ingress; 9421 break; 9422 case FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS: 9423 flower_handler = ice_setup_tc_block_cb_egress; 9424 break; 9425 default: 9426 return -EOPNOTSUPP; 9427 } 9428 9429 return flow_block_cb_setup_simple(type_data, 9430 &ice_block_cb_list, 9431 flower_handler, 9432 np, np, false); 9433 case TC_SETUP_QDISC_MQPRIO: 9434 if (ice_is_eswitch_mode_switchdev(pf)) { 9435 netdev_err(netdev, "TC MQPRIO offload not supported, switchdev is enabled\n"); 9436 return -EOPNOTSUPP; 9437 } 9438 9439 cdev = pf->cdev_info; 9440 if (cdev && cdev->adev) { 9441 mutex_lock(&pf->adev_mutex); 9442 device_lock(&cdev->adev->dev); 9443 locked = true; 9444 if (cdev->adev->dev.driver) { 9445 netdev_err(netdev, "Cannot change qdisc when RDMA is active\n"); 9446 err = -EBUSY; 9447 goto adev_unlock; 9448 } 9449 } 9450 9451 /* setup traffic classifier for receive side */ 9452 mutex_lock(&pf->tc_mutex); 9453 err = ice_setup_tc_mqprio_qdisc(netdev, type_data); 9454 mutex_unlock(&pf->tc_mutex); 9455 9456adev_unlock: 9457 if (locked) { 9458 device_unlock(&cdev->adev->dev); 9459 mutex_unlock(&pf->adev_mutex); 9460 } 9461 return err; 9462 case TC_SETUP_QDISC_ETF: 9463 return ice_offload_txtime(netdev, type_data); 9464 default: 9465 return -EOPNOTSUPP; 9466 } 9467 return -EOPNOTSUPP; 9468} 9469 9470static struct ice_indr_block_priv * 9471ice_indr_block_priv_lookup(struct ice_netdev_priv *np, 9472 struct net_device *netdev) 9473{ 9474 struct ice_indr_block_priv *cb_priv; 9475 9476 list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) { 9477 if (!cb_priv->netdev) 9478 return NULL; 9479 if (cb_priv->netdev == netdev) 9480 return cb_priv; 9481 } 9482 return NULL; 9483} 9484 9485static int 9486ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data, 9487 void *indr_priv) 9488{ 9489 struct ice_indr_block_priv *priv = indr_priv; 9490 struct ice_netdev_priv *np = priv->np; 9491 9492 switch (type) { 9493 case TC_SETUP_CLSFLOWER: 9494 return ice_setup_tc_cls_flower(np, priv->netdev, 9495 (struct flow_cls_offload *) 9496 type_data, false); 9497 default: 9498 return -EOPNOTSUPP; 9499 } 9500} 9501 9502static int 9503ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch, 9504 struct ice_netdev_priv *np, 9505 struct flow_block_offload *f, void *data, 9506 void (*cleanup)(struct flow_block_cb *block_cb)) 9507{ 9508 struct ice_indr_block_priv *indr_priv; 9509 struct flow_block_cb *block_cb; 9510 9511 if (!ice_is_tunnel_supported(netdev) && 9512 !(is_vlan_dev(netdev) && 9513 vlan_dev_real_dev(netdev) == np->vsi->netdev)) 9514 return -EOPNOTSUPP; 9515 9516 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 9517 return -EOPNOTSUPP; 9518 9519 switch (f->command) { 9520 case FLOW_BLOCK_BIND: 9521 indr_priv = ice_indr_block_priv_lookup(np, netdev); 9522 if (indr_priv) 9523 return -EEXIST; 9524 9525 indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL); 9526 if (!indr_priv) 9527 return -ENOMEM; 9528 9529 indr_priv->netdev = netdev; 9530 indr_priv->np = np; 9531 list_add(&indr_priv->list, &np->tc_indr_block_priv_list); 9532 9533 block_cb = 9534 flow_indr_block_cb_alloc(ice_indr_setup_block_cb, 9535 indr_priv, indr_priv, 9536 ice_rep_indr_tc_block_unbind, 9537 f, netdev, sch, data, np, 9538 cleanup); 9539 9540 if (IS_ERR(block_cb)) { 9541 list_del(&indr_priv->list); 9542 kfree(indr_priv); 9543 return PTR_ERR(block_cb); 9544 } 9545 flow_block_cb_add(block_cb, f); 9546 list_add_tail(&block_cb->driver_list, &ice_block_cb_list); 9547 break; 9548 case FLOW_BLOCK_UNBIND: 9549 indr_priv = ice_indr_block_priv_lookup(np, netdev); 9550 if (!indr_priv) 9551 return -ENOENT; 9552 9553 block_cb = flow_block_cb_lookup(f->block, 9554 ice_indr_setup_block_cb, 9555 indr_priv); 9556 if (!block_cb) 9557 return -ENOENT; 9558 9559 flow_indr_block_cb_remove(block_cb, f); 9560 9561 list_del(&block_cb->driver_list); 9562 break; 9563 default: 9564 return -EOPNOTSUPP; 9565 } 9566 return 0; 9567} 9568 9569static int 9570ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 9571 void *cb_priv, enum tc_setup_type type, void *type_data, 9572 void *data, 9573 void (*cleanup)(struct flow_block_cb *block_cb)) 9574{ 9575 switch (type) { 9576 case TC_SETUP_BLOCK: 9577 return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data, 9578 data, cleanup); 9579 9580 default: 9581 return -EOPNOTSUPP; 9582 } 9583} 9584 9585/** 9586 * ice_open - Called when a network interface becomes active 9587 * @netdev: network interface device structure 9588 * 9589 * The open entry point is called when a network interface is made 9590 * active by the system (IFF_UP). At this point all resources needed 9591 * for transmit and receive operations are allocated, the interrupt 9592 * handler is registered with the OS, the netdev watchdog is enabled, 9593 * and the stack is notified that the interface is ready. 9594 * 9595 * Returns 0 on success, negative value on failure 9596 */ 9597int ice_open(struct net_device *netdev) 9598{ 9599 struct ice_pf *pf = ice_netdev_to_pf(netdev); 9600 9601 if (ice_is_reset_in_progress(pf->state)) { 9602 netdev_err(netdev, "can't open net device while reset is in progress"); 9603 return -EBUSY; 9604 } 9605 9606 return ice_open_internal(netdev); 9607} 9608 9609/** 9610 * ice_open_internal - Called when a network interface becomes active 9611 * @netdev: network interface device structure 9612 * 9613 * Internal ice_open implementation. Should not be used directly except for ice_open and reset 9614 * handling routine 9615 * 9616 * Returns 0 on success, negative value on failure 9617 */ 9618int ice_open_internal(struct net_device *netdev) 9619{ 9620 struct ice_netdev_priv *np = netdev_priv(netdev); 9621 struct ice_vsi *vsi = np->vsi; 9622 struct ice_pf *pf = vsi->back; 9623 struct ice_port_info *pi; 9624 int err; 9625 9626 if (test_bit(ICE_NEEDS_RESTART, pf->state)) { 9627 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 9628 return -EIO; 9629 } 9630 9631 netif_carrier_off(netdev); 9632 9633 pi = vsi->port_info; 9634 err = ice_update_link_info(pi); 9635 if (err) { 9636 netdev_err(netdev, "Failed to get link info, error %d\n", err); 9637 return err; 9638 } 9639 9640 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 9641 9642 /* Set PHY if there is media, otherwise, turn off PHY */ 9643 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 9644 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 9645 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) { 9646 err = ice_init_phy_user_cfg(pi); 9647 if (err) { 9648 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 9649 err); 9650 return err; 9651 } 9652 } 9653 9654 err = ice_configure_phy(vsi); 9655 if (err) { 9656 netdev_err(netdev, "Failed to set physical link up, error %d\n", 9657 err); 9658 return err; 9659 } 9660 } else { 9661 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 9662 ice_set_link(vsi, false); 9663 } 9664 9665 err = ice_vsi_open(vsi); 9666 if (err) 9667 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 9668 vsi->vsi_num, vsi->vsw->sw_id); 9669 9670 return err; 9671} 9672 9673/** 9674 * ice_stop - Disables a network interface 9675 * @netdev: network interface device structure 9676 * 9677 * The stop entry point is called when an interface is de-activated by the OS, 9678 * and the netdevice enters the DOWN state. The hardware is still under the 9679 * driver's control, but the netdev interface is disabled. 9680 * 9681 * Returns success only - not allowed to fail 9682 */ 9683int ice_stop(struct net_device *netdev) 9684{ 9685 struct ice_netdev_priv *np = netdev_priv(netdev); 9686 struct ice_vsi *vsi = np->vsi; 9687 struct ice_pf *pf = vsi->back; 9688 9689 if (ice_is_reset_in_progress(pf->state)) { 9690 netdev_err(netdev, "can't stop net device while reset is in progress"); 9691 return -EBUSY; 9692 } 9693 9694 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 9695 int link_err = ice_force_phys_link_state(vsi, false); 9696 9697 if (link_err) { 9698 if (link_err == -ENOMEDIUM) 9699 netdev_info(vsi->netdev, "Skipping link reconfig - no media attached, VSI %d\n", 9700 vsi->vsi_num); 9701 else 9702 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 9703 vsi->vsi_num, link_err); 9704 9705 ice_vsi_close(vsi); 9706 return -EIO; 9707 } 9708 } 9709 9710 ice_vsi_close(vsi); 9711 9712 return 0; 9713} 9714 9715/** 9716 * ice_features_check - Validate encapsulated packet conforms to limits 9717 * @skb: skb buffer 9718 * @netdev: This port's netdev 9719 * @features: Offload features that the stack believes apply 9720 */ 9721static netdev_features_t 9722ice_features_check(struct sk_buff *skb, 9723 struct net_device __always_unused *netdev, 9724 netdev_features_t features) 9725{ 9726 bool gso = skb_is_gso(skb); 9727 size_t len; 9728 9729 /* No point in doing any of this if neither checksum nor GSO are 9730 * being requested for this frame. We can rule out both by just 9731 * checking for CHECKSUM_PARTIAL 9732 */ 9733 if (skb->ip_summed != CHECKSUM_PARTIAL) 9734 return features; 9735 9736 /* We cannot support GSO if the MSS is going to be less than 9737 * 64 bytes. If it is then we need to drop support for GSO. 9738 */ 9739 if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS)) 9740 features &= ~NETIF_F_GSO_MASK; 9741 9742 len = skb_network_offset(skb); 9743 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 9744 goto out_rm_features; 9745 9746 len = skb_network_header_len(skb); 9747 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 9748 goto out_rm_features; 9749 9750 if (skb->encapsulation) { 9751 /* this must work for VXLAN frames AND IPIP/SIT frames, and in 9752 * the case of IPIP frames, the transport header pointer is 9753 * after the inner header! So check to make sure that this 9754 * is a GRE or UDP_TUNNEL frame before doing that math. 9755 */ 9756 if (gso && (skb_shinfo(skb)->gso_type & 9757 (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) { 9758 len = skb_inner_network_header(skb) - 9759 skb_transport_header(skb); 9760 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 9761 goto out_rm_features; 9762 } 9763 9764 len = skb_inner_network_header_len(skb); 9765 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 9766 goto out_rm_features; 9767 } 9768 9769 return features; 9770out_rm_features: 9771 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9772} 9773 9774static const struct net_device_ops ice_netdev_safe_mode_ops = { 9775 .ndo_open = ice_open, 9776 .ndo_stop = ice_stop, 9777 .ndo_start_xmit = ice_start_xmit, 9778 .ndo_set_mac_address = ice_set_mac_address, 9779 .ndo_validate_addr = eth_validate_addr, 9780 .ndo_change_mtu = ice_change_mtu, 9781 .ndo_get_stats64 = ice_get_stats64, 9782 .ndo_tx_timeout = ice_tx_timeout, 9783 .ndo_bpf = ice_xdp_safe_mode, 9784}; 9785 9786static const struct net_device_ops ice_netdev_ops = { 9787 .ndo_open = ice_open, 9788 .ndo_stop = ice_stop, 9789 .ndo_start_xmit = ice_start_xmit, 9790 .ndo_select_queue = ice_select_queue, 9791 .ndo_features_check = ice_features_check, 9792 .ndo_fix_features = ice_fix_features, 9793 .ndo_set_rx_mode = ice_set_rx_mode, 9794 .ndo_set_mac_address = ice_set_mac_address, 9795 .ndo_validate_addr = eth_validate_addr, 9796 .ndo_change_mtu = ice_change_mtu, 9797 .ndo_get_stats64 = ice_get_stats64, 9798 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 9799 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 9800 .ndo_set_vf_mac = ice_set_vf_mac, 9801 .ndo_get_vf_config = ice_get_vf_cfg, 9802 .ndo_set_vf_trust = ice_set_vf_trust, 9803 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 9804 .ndo_set_vf_link_state = ice_set_vf_link_state, 9805 .ndo_get_vf_stats = ice_get_vf_stats, 9806 .ndo_set_vf_rate = ice_set_vf_bw, 9807 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 9808 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 9809 .ndo_setup_tc = ice_setup_tc, 9810 .ndo_set_features = ice_set_features, 9811 .ndo_bridge_getlink = ice_bridge_getlink, 9812 .ndo_bridge_setlink = ice_bridge_setlink, 9813 .ndo_fdb_add = ice_fdb_add, 9814 .ndo_fdb_del = ice_fdb_del, 9815#ifdef CONFIG_RFS_ACCEL 9816 .ndo_rx_flow_steer = ice_rx_flow_steer, 9817#endif 9818 .ndo_tx_timeout = ice_tx_timeout, 9819 .ndo_bpf = ice_xdp, 9820 .ndo_xdp_xmit = ice_xdp_xmit, 9821 .ndo_xsk_wakeup = ice_xsk_wakeup, 9822 .ndo_hwtstamp_get = ice_ptp_hwtstamp_get, 9823 .ndo_hwtstamp_set = ice_ptp_hwtstamp_set, 9824};
~/
Read-only file preview.