FILE
i40e_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/intel/i40e/i40e_main.c
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2013 - 2021 Intel Corporation. */ 3 4#include <generated/utsrelease.h> 5#include <linux/crash_dump.h> 6#include <linux/net/intel/libie/pctype.h> 7#include <linux/if_bridge.h> 8#include <linux/if_macvlan.h> 9#include <linux/module.h> 10#include <net/pkt_cls.h> 11#include <net/xdp_sock_drv.h> 12 13/* Local includes */ 14#include "i40e.h" 15#include "i40e_devids.h" 16#include "i40e_diag.h" 17#include "i40e_lan_hmc.h" 18#include "i40e_virtchnl_pf.h" 19#include "i40e_xsk.h" 20 21/* All i40e tracepoints are defined by the include below, which 22 * must be included exactly once across the whole kernel with 23 * CREATE_TRACE_POINTS defined 24 */ 25#define CREATE_TRACE_POINTS 26#include "i40e_trace.h" 27 28const char i40e_driver_name[] = "i40e"; 29static const char i40e_driver_string[] = 30 "Intel(R) Ethernet Connection XL710 Network Driver"; 31 32static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 33 34/* a bit of forward declarations */ 35static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 36static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 37static int i40e_add_vsi(struct i40e_vsi *vsi); 38static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 39static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired); 40static int i40e_setup_misc_vector(struct i40e_pf *pf); 41static void i40e_determine_queue_usage(struct i40e_pf *pf); 42static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 43static void i40e_prep_for_reset(struct i40e_pf *pf); 44static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 45 bool lock_acquired); 46static int i40e_reset(struct i40e_pf *pf); 47static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 48static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 49static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 50static bool i40e_check_recovery_mode(struct i40e_pf *pf); 51static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 52static void i40e_fdir_sb_setup(struct i40e_pf *pf); 53static int i40e_veb_get_bw_info(struct i40e_veb *veb); 54static int i40e_get_capabilities(struct i40e_pf *pf, 55 enum i40e_admin_queue_opc list_type); 56static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 57 58/* i40e_pci_tbl - PCI Device ID Table 59 * 60 * Last entry must be all 0s 61 * 62 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 63 * Class, Class Mask, private data (not used) } 64 */ 65static const struct pci_device_id i40e_pci_tbl[] = { 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 78 /* 79 * This ID conflicts with ipw2200, but the devices can be differentiated 80 * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200 81 * devices use PCI_CLASS_NETWORK_OTHER. 82 */ 83 {PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B), 84 PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0}, 92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 98 /* required last entry */ 99 {0, } 100}; 101MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 102 103#define I40E_MAX_VF_COUNT 128 104static int debug = -1; 105module_param(debug, uint, 0); 106MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 107 108MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 109MODULE_IMPORT_NS("LIBIE"); 110MODULE_IMPORT_NS("LIBIE_ADMINQ"); 111MODULE_LICENSE("GPL v2"); 112 113static struct workqueue_struct *i40e_wq; 114 115static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f, 116 struct net_device *netdev, int delta) 117{ 118 struct netdev_hw_addr_list *ha_list; 119 struct netdev_hw_addr *ha; 120 121 if (!f || !netdev) 122 return; 123 124 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr)) 125 ha_list = &netdev->uc; 126 else 127 ha_list = &netdev->mc; 128 129 netdev_hw_addr_list_for_each(ha, ha_list) { 130 if (ether_addr_equal(ha->addr, f->macaddr)) { 131 ha->refcount += delta; 132 if (ha->refcount <= 0) 133 ha->refcount = 1; 134 break; 135 } 136 } 137} 138 139/** 140 * i40e_hw_to_dev - get device pointer from the hardware structure 141 * @hw: pointer to the device HW structure 142 **/ 143struct device *i40e_hw_to_dev(struct i40e_hw *hw) 144{ 145 struct i40e_pf *pf = i40e_hw_to_pf(hw); 146 147 return &pf->pdev->dev; 148} 149 150/** 151 * i40e_allocate_dma_mem - OS specific memory alloc for shared code 152 * @hw: pointer to the HW structure 153 * @mem: ptr to mem struct to fill out 154 * @size: size of memory requested 155 * @alignment: what to align the allocation to 156 **/ 157int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem, 158 u64 size, u32 alignment) 159{ 160 struct i40e_pf *pf = i40e_hw_to_pf(hw); 161 162 mem->size = ALIGN(size, alignment); 163 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 164 GFP_KERNEL); 165 if (!mem->va) 166 return -ENOMEM; 167 168 return 0; 169} 170 171/** 172 * i40e_free_dma_mem - OS specific memory free for shared code 173 * @hw: pointer to the HW structure 174 * @mem: ptr to mem struct to free 175 **/ 176int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem) 177{ 178 struct i40e_pf *pf = i40e_hw_to_pf(hw); 179 180 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 181 mem->va = NULL; 182 mem->pa = 0; 183 mem->size = 0; 184 185 return 0; 186} 187 188/** 189 * i40e_allocate_virt_mem - OS specific memory alloc for shared code 190 * @hw: pointer to the HW structure 191 * @mem: ptr to mem struct to fill out 192 * @size: size of memory requested 193 **/ 194int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem, 195 u32 size) 196{ 197 mem->size = size; 198 mem->va = kzalloc(size, GFP_KERNEL); 199 200 if (!mem->va) 201 return -ENOMEM; 202 203 return 0; 204} 205 206/** 207 * i40e_free_virt_mem - OS specific memory free for shared code 208 * @hw: pointer to the HW structure 209 * @mem: ptr to mem struct to free 210 **/ 211int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem) 212{ 213 /* it's ok to kfree a NULL pointer */ 214 kfree(mem->va); 215 mem->va = NULL; 216 mem->size = 0; 217 218 return 0; 219} 220 221/** 222 * i40e_get_lump - find a lump of free generic resource 223 * @pf: board private structure 224 * @pile: the pile of resource to search 225 * @needed: the number of items needed 226 * @id: an owner id to stick on the items assigned 227 * 228 * Returns the base item index of the lump, or negative for error 229 **/ 230static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 231 u16 needed, u16 id) 232{ 233 int ret = -ENOMEM; 234 int i, j; 235 236 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 237 dev_info(&pf->pdev->dev, 238 "param err: pile=%s needed=%d id=0x%04x\n", 239 pile ? "<valid>" : "<null>", needed, id); 240 return -EINVAL; 241 } 242 243 /* Allocate last queue in the pile for FDIR VSI queue 244 * so it doesn't fragment the qp_pile 245 */ 246 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) { 247 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) { 248 dev_err(&pf->pdev->dev, 249 "Cannot allocate queue %d for I40E_VSI_FDIR\n", 250 pile->num_entries - 1); 251 return -ENOMEM; 252 } 253 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT; 254 return pile->num_entries - 1; 255 } 256 257 i = 0; 258 while (i < pile->num_entries) { 259 /* skip already allocated entries */ 260 if (pile->list[i] & I40E_PILE_VALID_BIT) { 261 i++; 262 continue; 263 } 264 265 /* do we have enough in this lump? */ 266 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 267 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 268 break; 269 } 270 271 if (j == needed) { 272 /* there was enough, so assign it to the requestor */ 273 for (j = 0; j < needed; j++) 274 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 275 ret = i; 276 break; 277 } 278 279 /* not enough, so skip over it and continue looking */ 280 i += j; 281 } 282 283 return ret; 284} 285 286/** 287 * i40e_put_lump - return a lump of generic resource 288 * @pile: the pile of resource to search 289 * @index: the base item index 290 * @id: the owner id of the items assigned 291 * 292 * Returns the count of items in the lump 293 **/ 294static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 295{ 296 int valid_id = (id | I40E_PILE_VALID_BIT); 297 int count = 0; 298 u16 i; 299 300 if (!pile || index >= pile->num_entries) 301 return -EINVAL; 302 303 for (i = index; 304 i < pile->num_entries && pile->list[i] == valid_id; 305 i++) { 306 pile->list[i] = 0; 307 count++; 308 } 309 310 311 return count; 312} 313 314/** 315 * i40e_find_vsi_from_id - searches for the vsi with the given id 316 * @pf: the pf structure to search for the vsi 317 * @id: id of the vsi it is searching for 318 **/ 319struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 320{ 321 struct i40e_vsi *vsi; 322 int i; 323 324 i40e_pf_for_each_vsi(pf, i, vsi) 325 if (vsi->id == id) 326 return vsi; 327 328 return NULL; 329} 330 331/** 332 * i40e_service_event_schedule - Schedule the service task to wake up 333 * @pf: board private structure 334 * 335 * If not already scheduled, this puts the task into the work queue 336 **/ 337void i40e_service_event_schedule(struct i40e_pf *pf) 338{ 339 if ((!test_bit(__I40E_DOWN, pf->state) && 340 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 341 test_bit(__I40E_RECOVERY_MODE, pf->state)) 342 queue_work(i40e_wq, &pf->service_task); 343} 344 345/** 346 * i40e_tx_timeout - Respond to a Tx Hang 347 * @netdev: network interface device structure 348 * @txqueue: queue number timing out 349 * 350 * If any port has noticed a Tx timeout, it is likely that the whole 351 * device is munged, not just the one netdev port, so go for the full 352 * reset. 353 **/ 354static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 355{ 356 struct i40e_netdev_priv *np = netdev_priv(netdev); 357 struct i40e_vsi *vsi = np->vsi; 358 struct i40e_pf *pf = vsi->back; 359 struct i40e_ring *tx_ring = NULL; 360 unsigned int i; 361 u32 head, val; 362 363 pf->tx_timeout_count++; 364 365 /* with txqueue index, find the tx_ring struct */ 366 for (i = 0; i < vsi->num_queue_pairs; i++) { 367 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 368 if (txqueue == 369 vsi->tx_rings[i]->queue_index) { 370 tx_ring = vsi->tx_rings[i]; 371 break; 372 } 373 } 374 } 375 376 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 377 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 378 else if (time_before(jiffies, 379 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 380 return; /* don't do any new action before the next timeout */ 381 382 /* don't kick off another recovery if one is already pending */ 383 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 384 return; 385 386 if (tx_ring) { 387 head = i40e_get_head(tx_ring); 388 /* Read interrupt register */ 389 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 390 val = rd32(&pf->hw, 391 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 392 tx_ring->vsi->base_vector - 1)); 393 else 394 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 395 396 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 397 vsi->seid, txqueue, tx_ring->next_to_clean, 398 head, tx_ring->next_to_use, 399 readl(tx_ring->tail), val); 400 } 401 402 pf->tx_timeout_last_recovery = jiffies; 403 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 404 pf->tx_timeout_recovery_level, txqueue); 405 406 switch (pf->tx_timeout_recovery_level) { 407 case 1: 408 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 409 break; 410 case 2: 411 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 412 break; 413 case 3: 414 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 415 break; 416 default: 417 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n"); 418 set_bit(__I40E_DOWN_REQUESTED, pf->state); 419 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state); 420 break; 421 } 422 423 i40e_service_event_schedule(pf); 424 pf->tx_timeout_recovery_level++; 425} 426 427/** 428 * i40e_get_vsi_stats_struct - Get System Network Statistics 429 * @vsi: the VSI we care about 430 * 431 * Returns the address of the device statistics structure. 432 * The statistics are actually updated from the service task. 433 **/ 434struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 435{ 436 return &vsi->net_stats; 437} 438 439/** 440 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 441 * @ring: Tx ring to get statistics from 442 * @stats: statistics entry to be updated 443 **/ 444static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 445 struct rtnl_link_stats64 *stats) 446{ 447 u64 bytes, packets; 448 unsigned int start; 449 450 do { 451 start = u64_stats_fetch_begin(&ring->syncp); 452 packets = ring->stats.packets; 453 bytes = ring->stats.bytes; 454 } while (u64_stats_fetch_retry(&ring->syncp, start)); 455 456 stats->tx_packets += packets; 457 stats->tx_bytes += bytes; 458} 459 460/** 461 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 462 * @netdev: network interface device structure 463 * @stats: data structure to store statistics 464 * 465 * Returns the address of the device statistics structure. 466 * The statistics are actually updated from the service task. 467 **/ 468static void i40e_get_netdev_stats_struct(struct net_device *netdev, 469 struct rtnl_link_stats64 *stats) 470{ 471 struct i40e_netdev_priv *np = netdev_priv(netdev); 472 struct i40e_vsi *vsi = np->vsi; 473 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 474 struct i40e_ring *ring; 475 int i; 476 477 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 478 return; 479 480 if (!vsi->tx_rings) 481 return; 482 483 rcu_read_lock(); 484 for (i = 0; i < vsi->num_queue_pairs; i++) { 485 u64 bytes, packets; 486 unsigned int start; 487 488 ring = READ_ONCE(vsi->tx_rings[i]); 489 if (!ring) 490 continue; 491 i40e_get_netdev_stats_struct_tx(ring, stats); 492 493 if (i40e_enabled_xdp_vsi(vsi)) { 494 ring = READ_ONCE(vsi->xdp_rings[i]); 495 if (!ring) 496 continue; 497 i40e_get_netdev_stats_struct_tx(ring, stats); 498 } 499 500 ring = READ_ONCE(vsi->rx_rings[i]); 501 if (!ring) 502 continue; 503 do { 504 start = u64_stats_fetch_begin(&ring->syncp); 505 packets = ring->stats.packets; 506 bytes = ring->stats.bytes; 507 } while (u64_stats_fetch_retry(&ring->syncp, start)); 508 509 stats->rx_packets += packets; 510 stats->rx_bytes += bytes; 511 512 } 513 rcu_read_unlock(); 514 515 /* following stats updated by i40e_watchdog_subtask() */ 516 stats->multicast = vsi_stats->multicast; 517 stats->tx_errors = vsi_stats->tx_errors; 518 stats->tx_dropped = vsi_stats->tx_dropped; 519 stats->rx_errors = vsi_stats->rx_errors; 520 stats->rx_dropped = vsi_stats->rx_dropped; 521 stats->rx_missed_errors = vsi_stats->rx_missed_errors; 522 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 523 stats->rx_length_errors = vsi_stats->rx_length_errors; 524} 525 526/** 527 * i40e_vsi_reset_stats - Resets all stats of the given vsi 528 * @vsi: the VSI to have its stats reset 529 **/ 530void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 531{ 532 struct rtnl_link_stats64 *ns; 533 int i; 534 535 if (!vsi) 536 return; 537 538 ns = i40e_get_vsi_stats_struct(vsi); 539 memset(ns, 0, sizeof(*ns)); 540 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 541 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 542 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 543 if (vsi->rx_rings && vsi->rx_rings[0]) { 544 for (i = 0; i < vsi->num_queue_pairs; i++) { 545 memset(&vsi->rx_rings[i]->stats, 0, 546 sizeof(vsi->rx_rings[i]->stats)); 547 memset(&vsi->rx_rings[i]->rx_stats, 0, 548 sizeof(vsi->rx_rings[i]->rx_stats)); 549 memset(&vsi->tx_rings[i]->stats, 0, 550 sizeof(vsi->tx_rings[i]->stats)); 551 memset(&vsi->tx_rings[i]->tx_stats, 0, 552 sizeof(vsi->tx_rings[i]->tx_stats)); 553 } 554 } 555 vsi->stat_offsets_loaded = false; 556} 557 558/** 559 * i40e_pf_reset_stats - Reset all of the stats for the given PF 560 * @pf: the PF to be reset 561 **/ 562void i40e_pf_reset_stats(struct i40e_pf *pf) 563{ 564 struct i40e_veb *veb; 565 int i; 566 567 memset(&pf->stats, 0, sizeof(pf->stats)); 568 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 569 pf->stat_offsets_loaded = false; 570 571 i40e_pf_for_each_veb(pf, i, veb) { 572 memset(&veb->stats, 0, sizeof(veb->stats)); 573 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets)); 574 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats)); 575 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets)); 576 veb->stat_offsets_loaded = false; 577 } 578 pf->hw_csum_rx_error = 0; 579} 580 581/** 582 * i40e_compute_pci_to_hw_id - compute index form PCI function. 583 * @vsi: ptr to the VSI to read from. 584 * @hw: ptr to the hardware info. 585 **/ 586static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw) 587{ 588 int pf_count = i40e_get_pf_count(hw); 589 590 if (vsi->type == I40E_VSI_SRIOV) 591 return (hw->port * BIT(7)) / pf_count + vsi->vf_id; 592 593 return hw->port + BIT(7); 594} 595 596/** 597 * i40e_stat_update64 - read and update a 64 bit stat from the chip. 598 * @hw: ptr to the hardware info. 599 * @hireg: the high 32 bit reg to read. 600 * @loreg: the low 32 bit reg to read. 601 * @offset_loaded: has the initial offset been loaded yet. 602 * @offset: ptr to current offset value. 603 * @stat: ptr to the stat. 604 * 605 * Since the device stats are not reset at PFReset, they will not 606 * be zeroed when the driver starts. We'll save the first values read 607 * and use them as offsets to be subtracted from the raw values in order 608 * to report stats that count from zero. 609 **/ 610static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg, 611 bool offset_loaded, u64 *offset, u64 *stat) 612{ 613 u64 new_data; 614 615 new_data = rd64(hw, loreg); 616 617 if (!offset_loaded || new_data < *offset) 618 *offset = new_data; 619 *stat = new_data - *offset; 620} 621 622/** 623 * i40e_stat_update48 - read and update a 48 bit stat from the chip 624 * @hw: ptr to the hardware info 625 * @hireg: the high 32 bit reg to read 626 * @loreg: the low 32 bit reg to read 627 * @offset_loaded: has the initial offset been loaded yet 628 * @offset: ptr to current offset value 629 * @stat: ptr to the stat 630 * 631 * Since the device stats are not reset at PFReset, they likely will not 632 * be zeroed when the driver starts. We'll save the first values read 633 * and use them as offsets to be subtracted from the raw values in order 634 * to report stats that count from zero. In the process, we also manage 635 * the potential roll-over. 636 **/ 637static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 638 bool offset_loaded, u64 *offset, u64 *stat) 639{ 640 u64 new_data; 641 642 if (hw->device_id == I40E_DEV_ID_QEMU) { 643 new_data = rd32(hw, loreg); 644 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 645 } else { 646 new_data = rd64(hw, loreg); 647 } 648 if (!offset_loaded) 649 *offset = new_data; 650 if (likely(new_data >= *offset)) 651 *stat = new_data - *offset; 652 else 653 *stat = (new_data + BIT_ULL(48)) - *offset; 654 *stat &= 0xFFFFFFFFFFFFULL; 655} 656 657/** 658 * i40e_stat_update32 - read and update a 32 bit stat from the chip 659 * @hw: ptr to the hardware info 660 * @reg: the hw reg to read 661 * @offset_loaded: has the initial offset been loaded yet 662 * @offset: ptr to current offset value 663 * @stat: ptr to the stat 664 **/ 665static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 666 bool offset_loaded, u64 *offset, u64 *stat) 667{ 668 u32 new_data; 669 670 new_data = rd32(hw, reg); 671 if (!offset_loaded) 672 *offset = new_data; 673 if (likely(new_data >= *offset)) 674 *stat = (u32)(new_data - *offset); 675 else 676 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 677} 678 679/** 680 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 681 * @hw: ptr to the hardware info 682 * @reg: the hw reg to read and clear 683 * @stat: ptr to the stat 684 **/ 685static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 686{ 687 u32 new_data = rd32(hw, reg); 688 689 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 690 *stat += new_data; 691} 692 693/** 694 * i40e_stats_update_rx_discards - update rx_discards. 695 * @vsi: ptr to the VSI to be updated. 696 * @hw: ptr to the hardware info. 697 * @stat_idx: VSI's stat_counter_idx. 698 * @offset_loaded: ptr to the VSI's stat_offsets_loaded. 699 * @stat_offset: ptr to stat_offset to store first read of specific register. 700 * @stat: ptr to VSI's stat to be updated. 701 **/ 702static void 703i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, 704 int stat_idx, bool offset_loaded, 705 struct i40e_eth_stats *stat_offset, 706 struct i40e_eth_stats *stat) 707{ 708 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, 709 &stat_offset->rx_discards, &stat->rx_discards); 710 i40e_stat_update64(hw, 711 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), 712 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), 713 offset_loaded, &stat_offset->rx_discards_other, 714 &stat->rx_discards_other); 715} 716 717/** 718 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 719 * @vsi: the VSI to be updated 720 **/ 721void i40e_update_eth_stats(struct i40e_vsi *vsi) 722{ 723 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 724 struct i40e_pf *pf = vsi->back; 725 struct i40e_hw *hw = &pf->hw; 726 struct i40e_eth_stats *oes; 727 struct i40e_eth_stats *es; /* device's eth stats */ 728 729 es = &vsi->eth_stats; 730 oes = &vsi->eth_stats_offsets; 731 732 /* Gather up the stats that the hw collects */ 733 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 734 vsi->stat_offsets_loaded, 735 &oes->tx_errors, &es->tx_errors); 736 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 737 vsi->stat_offsets_loaded, 738 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 739 740 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 741 I40E_GLV_GORCL(stat_idx), 742 vsi->stat_offsets_loaded, 743 &oes->rx_bytes, &es->rx_bytes); 744 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 745 I40E_GLV_UPRCL(stat_idx), 746 vsi->stat_offsets_loaded, 747 &oes->rx_unicast, &es->rx_unicast); 748 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 749 I40E_GLV_MPRCL(stat_idx), 750 vsi->stat_offsets_loaded, 751 &oes->rx_multicast, &es->rx_multicast); 752 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 753 I40E_GLV_BPRCL(stat_idx), 754 vsi->stat_offsets_loaded, 755 &oes->rx_broadcast, &es->rx_broadcast); 756 757 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 758 I40E_GLV_GOTCL(stat_idx), 759 vsi->stat_offsets_loaded, 760 &oes->tx_bytes, &es->tx_bytes); 761 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 762 I40E_GLV_UPTCL(stat_idx), 763 vsi->stat_offsets_loaded, 764 &oes->tx_unicast, &es->tx_unicast); 765 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 766 I40E_GLV_MPTCL(stat_idx), 767 vsi->stat_offsets_loaded, 768 &oes->tx_multicast, &es->tx_multicast); 769 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 770 I40E_GLV_BPTCL(stat_idx), 771 vsi->stat_offsets_loaded, 772 &oes->tx_broadcast, &es->tx_broadcast); 773 774 i40e_stats_update_rx_discards(vsi, hw, stat_idx, 775 vsi->stat_offsets_loaded, oes, es); 776 777 vsi->stat_offsets_loaded = true; 778} 779 780/** 781 * i40e_update_veb_stats - Update Switch component statistics 782 * @veb: the VEB being updated 783 **/ 784void i40e_update_veb_stats(struct i40e_veb *veb) 785{ 786 struct i40e_pf *pf = veb->pf; 787 struct i40e_hw *hw = &pf->hw; 788 struct i40e_eth_stats *oes; 789 struct i40e_eth_stats *es; /* device's eth stats */ 790 struct i40e_veb_tc_stats *veb_oes; 791 struct i40e_veb_tc_stats *veb_es; 792 int i, idx = 0; 793 794 idx = veb->stats_idx; 795 es = &veb->stats; 796 oes = &veb->stats_offsets; 797 veb_es = &veb->tc_stats; 798 veb_oes = &veb->tc_stats_offsets; 799 800 /* Gather up the stats that the hw collects */ 801 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 802 veb->stat_offsets_loaded, 803 &oes->tx_discards, &es->tx_discards); 804 if (hw->revision_id > 0) 805 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 806 veb->stat_offsets_loaded, 807 &oes->rx_unknown_protocol, 808 &es->rx_unknown_protocol); 809 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 810 veb->stat_offsets_loaded, 811 &oes->rx_bytes, &es->rx_bytes); 812 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 813 veb->stat_offsets_loaded, 814 &oes->rx_unicast, &es->rx_unicast); 815 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 816 veb->stat_offsets_loaded, 817 &oes->rx_multicast, &es->rx_multicast); 818 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 819 veb->stat_offsets_loaded, 820 &oes->rx_broadcast, &es->rx_broadcast); 821 822 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 823 veb->stat_offsets_loaded, 824 &oes->tx_bytes, &es->tx_bytes); 825 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 826 veb->stat_offsets_loaded, 827 &oes->tx_unicast, &es->tx_unicast); 828 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 829 veb->stat_offsets_loaded, 830 &oes->tx_multicast, &es->tx_multicast); 831 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 832 veb->stat_offsets_loaded, 833 &oes->tx_broadcast, &es->tx_broadcast); 834 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 835 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 836 I40E_GLVEBTC_RPCL(i, idx), 837 veb->stat_offsets_loaded, 838 &veb_oes->tc_rx_packets[i], 839 &veb_es->tc_rx_packets[i]); 840 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 841 I40E_GLVEBTC_RBCL(i, idx), 842 veb->stat_offsets_loaded, 843 &veb_oes->tc_rx_bytes[i], 844 &veb_es->tc_rx_bytes[i]); 845 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 846 I40E_GLVEBTC_TPCL(i, idx), 847 veb->stat_offsets_loaded, 848 &veb_oes->tc_tx_packets[i], 849 &veb_es->tc_tx_packets[i]); 850 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 851 I40E_GLVEBTC_TBCL(i, idx), 852 veb->stat_offsets_loaded, 853 &veb_oes->tc_tx_bytes[i], 854 &veb_es->tc_tx_bytes[i]); 855 } 856 veb->stat_offsets_loaded = true; 857} 858 859/** 860 * i40e_update_vsi_stats - Update the vsi statistics counters. 861 * @vsi: the VSI to be updated 862 * 863 * There are a few instances where we store the same stat in a 864 * couple of different structs. This is partly because we have 865 * the netdev stats that need to be filled out, which is slightly 866 * different from the "eth_stats" defined by the chip and used in 867 * VF communications. We sort it out here. 868 **/ 869static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 870{ 871 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy; 872 struct i40e_pf *pf = vsi->back; 873 struct rtnl_link_stats64 *ons; 874 struct rtnl_link_stats64 *ns; /* netdev stats */ 875 struct i40e_eth_stats *oes; 876 struct i40e_eth_stats *es; /* device's eth stats */ 877 u64 tx_restart, tx_busy; 878 struct i40e_ring *p; 879 u64 bytes, packets; 880 unsigned int start; 881 u64 tx_linearize; 882 u64 tx_force_wb; 883 u64 tx_stopped; 884 u64 rx_p, rx_b; 885 u64 tx_p, tx_b; 886 u16 q; 887 888 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 889 test_bit(__I40E_CONFIG_BUSY, pf->state)) 890 return; 891 892 ns = i40e_get_vsi_stats_struct(vsi); 893 ons = &vsi->net_stats_offsets; 894 es = &vsi->eth_stats; 895 oes = &vsi->eth_stats_offsets; 896 897 /* Gather up the netdev and vsi stats that the driver collects 898 * on the fly during packet processing 899 */ 900 rx_b = rx_p = 0; 901 tx_b = tx_p = 0; 902 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 903 tx_stopped = 0; 904 rx_page = 0; 905 rx_buf = 0; 906 rx_reuse = 0; 907 rx_alloc = 0; 908 rx_waive = 0; 909 rx_busy = 0; 910 rcu_read_lock(); 911 for (q = 0; q < vsi->num_queue_pairs; q++) { 912 /* locate Tx ring */ 913 p = READ_ONCE(vsi->tx_rings[q]); 914 if (!p) 915 continue; 916 917 do { 918 start = u64_stats_fetch_begin(&p->syncp); 919 packets = p->stats.packets; 920 bytes = p->stats.bytes; 921 } while (u64_stats_fetch_retry(&p->syncp, start)); 922 tx_b += bytes; 923 tx_p += packets; 924 tx_restart += p->tx_stats.restart_queue; 925 tx_busy += p->tx_stats.tx_busy; 926 tx_linearize += p->tx_stats.tx_linearize; 927 tx_force_wb += p->tx_stats.tx_force_wb; 928 tx_stopped += p->tx_stats.tx_stopped; 929 930 /* locate Rx ring */ 931 p = READ_ONCE(vsi->rx_rings[q]); 932 if (!p) 933 continue; 934 935 do { 936 start = u64_stats_fetch_begin(&p->syncp); 937 packets = p->stats.packets; 938 bytes = p->stats.bytes; 939 } while (u64_stats_fetch_retry(&p->syncp, start)); 940 rx_b += bytes; 941 rx_p += packets; 942 rx_buf += p->rx_stats.alloc_buff_failed; 943 rx_page += p->rx_stats.alloc_page_failed; 944 rx_reuse += p->rx_stats.page_reuse_count; 945 rx_alloc += p->rx_stats.page_alloc_count; 946 rx_waive += p->rx_stats.page_waive_count; 947 rx_busy += p->rx_stats.page_busy_count; 948 949 if (i40e_enabled_xdp_vsi(vsi)) { 950 /* locate XDP ring */ 951 p = READ_ONCE(vsi->xdp_rings[q]); 952 if (!p) 953 continue; 954 955 do { 956 start = u64_stats_fetch_begin(&p->syncp); 957 packets = p->stats.packets; 958 bytes = p->stats.bytes; 959 } while (u64_stats_fetch_retry(&p->syncp, start)); 960 tx_b += bytes; 961 tx_p += packets; 962 tx_restart += p->tx_stats.restart_queue; 963 tx_busy += p->tx_stats.tx_busy; 964 tx_linearize += p->tx_stats.tx_linearize; 965 tx_force_wb += p->tx_stats.tx_force_wb; 966 } 967 } 968 rcu_read_unlock(); 969 vsi->tx_restart = tx_restart; 970 vsi->tx_busy = tx_busy; 971 vsi->tx_linearize = tx_linearize; 972 vsi->tx_force_wb = tx_force_wb; 973 vsi->tx_stopped = tx_stopped; 974 vsi->rx_page_failed = rx_page; 975 vsi->rx_buf_failed = rx_buf; 976 vsi->rx_page_reuse = rx_reuse; 977 vsi->rx_page_alloc = rx_alloc; 978 vsi->rx_page_waive = rx_waive; 979 vsi->rx_page_busy = rx_busy; 980 981 ns->rx_packets = rx_p; 982 ns->rx_bytes = rx_b; 983 ns->tx_packets = tx_p; 984 ns->tx_bytes = tx_b; 985 986 /* update netdev stats from eth stats */ 987 i40e_update_eth_stats(vsi); 988 ons->tx_errors = oes->tx_errors; 989 ns->tx_errors = es->tx_errors; 990 ons->multicast = oes->rx_multicast; 991 ns->multicast = es->rx_multicast; 992 ons->rx_dropped = oes->rx_discards_other; 993 ns->rx_dropped = es->rx_discards_other; 994 ons->rx_missed_errors = oes->rx_discards; 995 ns->rx_missed_errors = es->rx_discards; 996 ons->tx_dropped = oes->tx_discards; 997 ns->tx_dropped = es->tx_discards; 998 999 /* pull in a couple PF stats if this is the main vsi */ 1000 if (vsi->type == I40E_VSI_MAIN) { 1001 ns->rx_crc_errors = pf->stats.crc_errors; 1002 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 1003 ns->rx_length_errors = pf->stats.rx_length_errors; 1004 } 1005} 1006 1007/** 1008 * i40e_update_pf_stats - Update the PF statistics counters. 1009 * @pf: the PF to be updated 1010 **/ 1011static void i40e_update_pf_stats(struct i40e_pf *pf) 1012{ 1013 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 1014 struct i40e_hw_port_stats *nsd = &pf->stats; 1015 struct i40e_hw *hw = &pf->hw; 1016 u32 val; 1017 int i; 1018 1019 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 1020 I40E_GLPRT_GORCL(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 1023 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 1024 I40E_GLPRT_GOTCL(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 1027 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->eth.rx_discards, 1030 &nsd->eth.rx_discards); 1031 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 1032 I40E_GLPRT_UPRCL(hw->port), 1033 pf->stat_offsets_loaded, 1034 &osd->eth.rx_unicast, 1035 &nsd->eth.rx_unicast); 1036 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 1037 I40E_GLPRT_MPRCL(hw->port), 1038 pf->stat_offsets_loaded, 1039 &osd->eth.rx_multicast, 1040 &nsd->eth.rx_multicast); 1041 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 1042 I40E_GLPRT_BPRCL(hw->port), 1043 pf->stat_offsets_loaded, 1044 &osd->eth.rx_broadcast, 1045 &nsd->eth.rx_broadcast); 1046 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 1047 I40E_GLPRT_UPTCL(hw->port), 1048 pf->stat_offsets_loaded, 1049 &osd->eth.tx_unicast, 1050 &nsd->eth.tx_unicast); 1051 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 1052 I40E_GLPRT_MPTCL(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->eth.tx_multicast, 1055 &nsd->eth.tx_multicast); 1056 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 1057 I40E_GLPRT_BPTCL(hw->port), 1058 pf->stat_offsets_loaded, 1059 &osd->eth.tx_broadcast, 1060 &nsd->eth.tx_broadcast); 1061 1062 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 1063 pf->stat_offsets_loaded, 1064 &osd->tx_dropped_link_down, 1065 &nsd->tx_dropped_link_down); 1066 1067 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 1068 pf->stat_offsets_loaded, 1069 &osd->crc_errors, &nsd->crc_errors); 1070 1071 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 1072 pf->stat_offsets_loaded, 1073 &osd->illegal_bytes, &nsd->illegal_bytes); 1074 1075 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 1076 pf->stat_offsets_loaded, 1077 &osd->mac_local_faults, 1078 &nsd->mac_local_faults); 1079 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 1080 pf->stat_offsets_loaded, 1081 &osd->mac_remote_faults, 1082 &nsd->mac_remote_faults); 1083 1084 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 1085 pf->stat_offsets_loaded, 1086 &osd->rx_length_errors, 1087 &nsd->rx_length_errors); 1088 1089 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 1090 pf->stat_offsets_loaded, 1091 &osd->link_xon_rx, &nsd->link_xon_rx); 1092 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 1093 pf->stat_offsets_loaded, 1094 &osd->link_xon_tx, &nsd->link_xon_tx); 1095 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 1096 pf->stat_offsets_loaded, 1097 &osd->link_xoff_rx, &nsd->link_xoff_rx); 1098 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 1099 pf->stat_offsets_loaded, 1100 &osd->link_xoff_tx, &nsd->link_xoff_tx); 1101 1102 for (i = 0; i < 8; i++) { 1103 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 1104 pf->stat_offsets_loaded, 1105 &osd->priority_xoff_rx[i], 1106 &nsd->priority_xoff_rx[i]); 1107 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 1108 pf->stat_offsets_loaded, 1109 &osd->priority_xon_rx[i], 1110 &nsd->priority_xon_rx[i]); 1111 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 1112 pf->stat_offsets_loaded, 1113 &osd->priority_xon_tx[i], 1114 &nsd->priority_xon_tx[i]); 1115 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 1116 pf->stat_offsets_loaded, 1117 &osd->priority_xoff_tx[i], 1118 &nsd->priority_xoff_tx[i]); 1119 i40e_stat_update32(hw, 1120 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1121 pf->stat_offsets_loaded, 1122 &osd->priority_xon_2_xoff[i], 1123 &nsd->priority_xon_2_xoff[i]); 1124 } 1125 1126 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1127 I40E_GLPRT_PRC64L(hw->port), 1128 pf->stat_offsets_loaded, 1129 &osd->rx_size_64, &nsd->rx_size_64); 1130 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1131 I40E_GLPRT_PRC127L(hw->port), 1132 pf->stat_offsets_loaded, 1133 &osd->rx_size_127, &nsd->rx_size_127); 1134 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1135 I40E_GLPRT_PRC255L(hw->port), 1136 pf->stat_offsets_loaded, 1137 &osd->rx_size_255, &nsd->rx_size_255); 1138 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1139 I40E_GLPRT_PRC511L(hw->port), 1140 pf->stat_offsets_loaded, 1141 &osd->rx_size_511, &nsd->rx_size_511); 1142 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1143 I40E_GLPRT_PRC1023L(hw->port), 1144 pf->stat_offsets_loaded, 1145 &osd->rx_size_1023, &nsd->rx_size_1023); 1146 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1147 I40E_GLPRT_PRC1522L(hw->port), 1148 pf->stat_offsets_loaded, 1149 &osd->rx_size_1522, &nsd->rx_size_1522); 1150 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1151 I40E_GLPRT_PRC9522L(hw->port), 1152 pf->stat_offsets_loaded, 1153 &osd->rx_size_big, &nsd->rx_size_big); 1154 1155 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1156 I40E_GLPRT_PTC64L(hw->port), 1157 pf->stat_offsets_loaded, 1158 &osd->tx_size_64, &nsd->tx_size_64); 1159 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1160 I40E_GLPRT_PTC127L(hw->port), 1161 pf->stat_offsets_loaded, 1162 &osd->tx_size_127, &nsd->tx_size_127); 1163 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1164 I40E_GLPRT_PTC255L(hw->port), 1165 pf->stat_offsets_loaded, 1166 &osd->tx_size_255, &nsd->tx_size_255); 1167 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1168 I40E_GLPRT_PTC511L(hw->port), 1169 pf->stat_offsets_loaded, 1170 &osd->tx_size_511, &nsd->tx_size_511); 1171 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1172 I40E_GLPRT_PTC1023L(hw->port), 1173 pf->stat_offsets_loaded, 1174 &osd->tx_size_1023, &nsd->tx_size_1023); 1175 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1176 I40E_GLPRT_PTC1522L(hw->port), 1177 pf->stat_offsets_loaded, 1178 &osd->tx_size_1522, &nsd->tx_size_1522); 1179 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1180 I40E_GLPRT_PTC9522L(hw->port), 1181 pf->stat_offsets_loaded, 1182 &osd->tx_size_big, &nsd->tx_size_big); 1183 1184 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1185 pf->stat_offsets_loaded, 1186 &osd->rx_undersize, &nsd->rx_undersize); 1187 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1188 pf->stat_offsets_loaded, 1189 &osd->rx_fragments, &nsd->rx_fragments); 1190 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1191 pf->stat_offsets_loaded, 1192 &osd->rx_oversize, &nsd->rx_oversize); 1193 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1194 pf->stat_offsets_loaded, 1195 &osd->rx_jabber, &nsd->rx_jabber); 1196 1197 /* FDIR stats */ 1198 i40e_stat_update_and_clear32(hw, 1199 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1200 &nsd->fd_atr_match); 1201 i40e_stat_update_and_clear32(hw, 1202 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1203 &nsd->fd_sb_match); 1204 i40e_stat_update_and_clear32(hw, 1205 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1206 &nsd->fd_atr_tunnel_match); 1207 1208 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1209 nsd->tx_lpi_status = 1210 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val); 1211 nsd->rx_lpi_status = 1212 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val); 1213 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1214 pf->stat_offsets_loaded, 1215 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1216 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1217 pf->stat_offsets_loaded, 1218 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1219 1220 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 1221 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1222 nsd->fd_sb_status = true; 1223 else 1224 nsd->fd_sb_status = false; 1225 1226 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 1227 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1228 nsd->fd_atr_status = true; 1229 else 1230 nsd->fd_atr_status = false; 1231 1232 pf->stat_offsets_loaded = true; 1233} 1234 1235/** 1236 * i40e_update_stats - Update the various statistics counters. 1237 * @vsi: the VSI to be updated 1238 * 1239 * Update the various stats for this VSI and its related entities. 1240 **/ 1241void i40e_update_stats(struct i40e_vsi *vsi) 1242{ 1243 struct i40e_pf *pf = vsi->back; 1244 1245 if (vsi->type == I40E_VSI_MAIN) 1246 i40e_update_pf_stats(pf); 1247 1248 i40e_update_vsi_stats(vsi); 1249} 1250 1251/** 1252 * i40e_count_all_filters - counts VSI MAC filters 1253 * @vsi: the VSI to be searched 1254 * 1255 * Return: count of MAC filters in any state. 1256 */ 1257int i40e_count_all_filters(struct i40e_vsi *vsi) 1258{ 1259 struct i40e_mac_filter *f; 1260 struct hlist_node *h; 1261 int bkt, cnt = 0; 1262 1263 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1264 cnt++; 1265 1266 return cnt; 1267} 1268 1269/** 1270 * i40e_count_active_filters - counts VSI MAC filters 1271 * @vsi: the VSI to be searched 1272 * 1273 * Return: count of active MAC filters. 1274 */ 1275int i40e_count_active_filters(struct i40e_vsi *vsi) 1276{ 1277 struct i40e_mac_filter *f; 1278 struct hlist_node *h; 1279 int bkt; 1280 int cnt = 0; 1281 1282 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1283 if (f->state == I40E_FILTER_NEW || 1284 f->state == I40E_FILTER_NEW_SYNC || 1285 f->state == I40E_FILTER_ACTIVE) 1286 ++cnt; 1287 } 1288 1289 return cnt; 1290} 1291 1292/** 1293 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1294 * @vsi: the VSI to be searched 1295 * @macaddr: the MAC address 1296 * @vlan: the vlan 1297 * 1298 * Returns ptr to the filter object or NULL 1299 **/ 1300static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1301 const u8 *macaddr, s16 vlan) 1302{ 1303 struct i40e_mac_filter *f; 1304 u64 key; 1305 1306 if (!vsi || !macaddr) 1307 return NULL; 1308 1309 key = i40e_addr_to_hkey(macaddr); 1310 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1311 if ((ether_addr_equal(macaddr, f->macaddr)) && 1312 (vlan == f->vlan)) 1313 return f; 1314 } 1315 return NULL; 1316} 1317 1318/** 1319 * i40e_find_mac - Find a mac addr in the macvlan filters list 1320 * @vsi: the VSI to be searched 1321 * @macaddr: the MAC address we are searching for 1322 * 1323 * Returns the first filter with the provided MAC address or NULL if 1324 * MAC address was not found 1325 **/ 1326struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1327{ 1328 struct i40e_mac_filter *f; 1329 u64 key; 1330 1331 if (!vsi || !macaddr) 1332 return NULL; 1333 1334 key = i40e_addr_to_hkey(macaddr); 1335 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1336 if ((ether_addr_equal(macaddr, f->macaddr))) 1337 return f; 1338 } 1339 return NULL; 1340} 1341 1342/** 1343 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1344 * @vsi: the VSI to be searched 1345 * 1346 * Returns true if VSI is in vlan mode or false otherwise 1347 **/ 1348bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1349{ 1350 /* If we have a PVID, always operate in VLAN mode */ 1351 if (vsi->info.pvid) 1352 return true; 1353 1354 /* We need to operate in VLAN mode whenever we have any filters with 1355 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1356 * time, incurring search cost repeatedly. However, we can notice two 1357 * things: 1358 * 1359 * 1) the only place where we can gain a VLAN filter is in 1360 * i40e_add_filter. 1361 * 1362 * 2) the only place where filters are actually removed is in 1363 * i40e_sync_filters_subtask. 1364 * 1365 * Thus, we can simply use a boolean value, has_vlan_filters which we 1366 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1367 * we have to perform the full search after deleting filters in 1368 * i40e_sync_filters_subtask, but we already have to search 1369 * filters here and can perform the check at the same time. This 1370 * results in avoiding embedding a loop for VLAN mode inside another 1371 * loop over all the filters, and should maintain correctness as noted 1372 * above. 1373 */ 1374 return vsi->has_vlan_filter; 1375} 1376 1377/** 1378 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1379 * @vsi: the VSI to configure 1380 * @tmp_add_list: list of filters ready to be added 1381 * @tmp_del_list: list of filters ready to be deleted 1382 * @vlan_filters: the number of active VLAN filters 1383 * 1384 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1385 * behave as expected. If we have any active VLAN filters remaining or about 1386 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1387 * so that they only match against untagged traffic. If we no longer have any 1388 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1389 * so that they match against both tagged and untagged traffic. In this way, 1390 * we ensure that we correctly receive the desired traffic. This ensures that 1391 * when we have an active VLAN we will receive only untagged traffic and 1392 * traffic matching active VLANs. If we have no active VLANs then we will 1393 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1394 * 1395 * Finally, in a similar fashion, this function also corrects filters when 1396 * there is an active PVID assigned to this VSI. 1397 * 1398 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1399 * 1400 * This function is only expected to be called from within 1401 * i40e_sync_vsi_filters. 1402 * 1403 * NOTE: This function expects to be called while under the 1404 * mac_filter_hash_lock 1405 */ 1406static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1407 struct hlist_head *tmp_add_list, 1408 struct hlist_head *tmp_del_list, 1409 int vlan_filters) 1410{ 1411 s16 pvid = le16_to_cpu(vsi->info.pvid); 1412 struct i40e_mac_filter *f, *add_head; 1413 struct i40e_new_mac_filter *new; 1414 struct hlist_node *h; 1415 int bkt, new_vlan; 1416 1417 /* To determine if a particular filter needs to be replaced we 1418 * have the three following conditions: 1419 * 1420 * a) if we have a PVID assigned, then all filters which are 1421 * not marked as VLAN=PVID must be replaced with filters that 1422 * are. 1423 * b) otherwise, if we have any active VLANS, all filters 1424 * which are marked as VLAN=-1 must be replaced with 1425 * filters marked as VLAN=0 1426 * c) finally, if we do not have any active VLANS, all filters 1427 * which are marked as VLAN=0 must be replaced with filters 1428 * marked as VLAN=-1 1429 */ 1430 1431 /* Update the filters about to be added in place */ 1432 hlist_for_each_entry(new, tmp_add_list, hlist) { 1433 if (pvid && new->f->vlan != pvid) 1434 new->f->vlan = pvid; 1435 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1436 new->f->vlan = 0; 1437 else if (!vlan_filters && new->f->vlan == 0) 1438 new->f->vlan = I40E_VLAN_ANY; 1439 } 1440 1441 /* Update the remaining active filters */ 1442 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1443 /* Combine the checks for whether a filter needs to be changed 1444 * and then determine the new VLAN inside the if block, in 1445 * order to avoid duplicating code for adding the new filter 1446 * then deleting the old filter. 1447 */ 1448 if ((pvid && f->vlan != pvid) || 1449 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1450 (!vlan_filters && f->vlan == 0)) { 1451 /* Determine the new vlan we will be adding */ 1452 if (pvid) 1453 new_vlan = pvid; 1454 else if (vlan_filters) 1455 new_vlan = 0; 1456 else 1457 new_vlan = I40E_VLAN_ANY; 1458 1459 /* Create the new filter */ 1460 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1461 if (!add_head) 1462 return -ENOMEM; 1463 1464 /* Create a temporary i40e_new_mac_filter */ 1465 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1466 if (!new) 1467 return -ENOMEM; 1468 1469 new->f = add_head; 1470 new->state = add_head->state; 1471 if (add_head->state == I40E_FILTER_NEW) 1472 add_head->state = I40E_FILTER_NEW_SYNC; 1473 1474 /* Add the new filter to the tmp list */ 1475 hlist_add_head(&new->hlist, tmp_add_list); 1476 1477 /* Put the original filter into the delete list */ 1478 f->state = I40E_FILTER_REMOVE; 1479 hash_del(&f->hlist); 1480 hlist_add_head(&f->hlist, tmp_del_list); 1481 } 1482 } 1483 1484 vsi->has_vlan_filter = !!vlan_filters; 1485 1486 return 0; 1487} 1488 1489/** 1490 * i40e_get_vf_new_vlan - Get new vlan id on a vf 1491 * @vsi: the vsi to configure 1492 * @new_mac: new mac filter to be added 1493 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL 1494 * @vlan_filters: the number of active VLAN filters 1495 * @trusted: flag if the VF is trusted 1496 * 1497 * Get new VLAN id based on current VLAN filters, trust, PVID 1498 * and vf-vlan-prune-disable flag. 1499 * 1500 * Returns the value of the new vlan filter or 1501 * the old value if no new filter is needed. 1502 */ 1503static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi, 1504 struct i40e_new_mac_filter *new_mac, 1505 struct i40e_mac_filter *f, 1506 int vlan_filters, 1507 bool trusted) 1508{ 1509 s16 pvid = le16_to_cpu(vsi->info.pvid); 1510 struct i40e_pf *pf = vsi->back; 1511 bool is_any; 1512 1513 if (new_mac) 1514 f = new_mac->f; 1515 1516 if (pvid && f->vlan != pvid) 1517 return pvid; 1518 1519 is_any = (trusted || 1520 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags)); 1521 1522 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) || 1523 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) || 1524 (is_any && !vlan_filters && f->vlan == 0)) { 1525 if (is_any) 1526 return I40E_VLAN_ANY; 1527 else 1528 return 0; 1529 } 1530 1531 return f->vlan; 1532} 1533 1534/** 1535 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary 1536 * @vsi: the vsi to configure 1537 * @tmp_add_list: list of filters ready to be added 1538 * @tmp_del_list: list of filters ready to be deleted 1539 * @vlan_filters: the number of active VLAN filters 1540 * @trusted: flag if the VF is trusted 1541 * 1542 * Correct VF VLAN filters based on current VLAN filters, trust, PVID 1543 * and vf-vlan-prune-disable flag. 1544 * 1545 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1546 * 1547 * This function is only expected to be called from within 1548 * i40e_sync_vsi_filters. 1549 * 1550 * NOTE: This function expects to be called while under the 1551 * mac_filter_hash_lock 1552 */ 1553static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi, 1554 struct hlist_head *tmp_add_list, 1555 struct hlist_head *tmp_del_list, 1556 int vlan_filters, 1557 bool trusted) 1558{ 1559 struct i40e_mac_filter *f, *add_head; 1560 struct i40e_new_mac_filter *new_mac; 1561 struct hlist_node *h; 1562 int bkt, new_vlan; 1563 1564 hlist_for_each_entry(new_mac, tmp_add_list, hlist) { 1565 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL, 1566 vlan_filters, trusted); 1567 } 1568 1569 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1570 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters, 1571 trusted); 1572 if (new_vlan != f->vlan) { 1573 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1574 if (!add_head) 1575 return -ENOMEM; 1576 /* Create a temporary i40e_new_mac_filter */ 1577 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC); 1578 if (!new_mac) 1579 return -ENOMEM; 1580 new_mac->f = add_head; 1581 new_mac->state = add_head->state; 1582 if (add_head->state == I40E_FILTER_NEW) 1583 add_head->state = I40E_FILTER_NEW_SYNC; 1584 1585 /* Add the new filter to the tmp list */ 1586 hlist_add_head(&new_mac->hlist, tmp_add_list); 1587 1588 /* Put the original filter into the delete list */ 1589 f->state = I40E_FILTER_REMOVE; 1590 hash_del(&f->hlist); 1591 hlist_add_head(&f->hlist, tmp_del_list); 1592 } 1593 } 1594 1595 vsi->has_vlan_filter = !!vlan_filters; 1596 return 0; 1597} 1598 1599/** 1600 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1601 * @vsi: the PF Main VSI - inappropriate for any other VSI 1602 * @macaddr: the MAC address 1603 * 1604 * Remove whatever filter the firmware set up so the driver can manage 1605 * its own filtering intelligently. 1606 **/ 1607static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1608{ 1609 struct i40e_aqc_remove_macvlan_element_data element; 1610 struct i40e_pf *pf = vsi->back; 1611 1612 /* Only appropriate for the PF main VSI */ 1613 if (vsi->type != I40E_VSI_MAIN) 1614 return; 1615 1616 memset(&element, 0, sizeof(element)); 1617 ether_addr_copy(element.mac_addr, macaddr); 1618 element.vlan_tag = 0; 1619 /* Ignore error returns, some firmware does it this way... */ 1620 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1621 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1622 1623 memset(&element, 0, sizeof(element)); 1624 ether_addr_copy(element.mac_addr, macaddr); 1625 element.vlan_tag = 0; 1626 /* ...and some firmware does it this way. */ 1627 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1628 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1629 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1630} 1631 1632/** 1633 * i40e_add_filter - Add a mac/vlan filter to the VSI 1634 * @vsi: the VSI to be searched 1635 * @macaddr: the MAC address 1636 * @vlan: the vlan 1637 * 1638 * Returns ptr to the filter object or NULL when no memory available. 1639 * 1640 * NOTE: This function is expected to be called with mac_filter_hash_lock 1641 * being held. 1642 **/ 1643struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1644 const u8 *macaddr, s16 vlan) 1645{ 1646 struct i40e_mac_filter *f; 1647 u64 key; 1648 1649 if (!vsi || !macaddr) 1650 return NULL; 1651 1652 f = i40e_find_filter(vsi, macaddr, vlan); 1653 if (!f) { 1654 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1655 if (!f) 1656 return NULL; 1657 1658 /* Update the boolean indicating if we need to function in 1659 * VLAN mode. 1660 */ 1661 if (vlan >= 0) 1662 vsi->has_vlan_filter = true; 1663 1664 ether_addr_copy(f->macaddr, macaddr); 1665 f->vlan = vlan; 1666 f->state = I40E_FILTER_NEW; 1667 INIT_HLIST_NODE(&f->hlist); 1668 1669 key = i40e_addr_to_hkey(macaddr); 1670 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1671 1672 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1673 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1674 } 1675 1676 /* If we're asked to add a filter that has been marked for removal, it 1677 * is safe to simply restore it to active state. __i40e_del_filter 1678 * will have simply deleted any filters which were previously marked 1679 * NEW or FAILED, so if it is currently marked REMOVE it must have 1680 * previously been ACTIVE. Since we haven't yet run the sync filters 1681 * task, just restore this filter to the ACTIVE state so that the 1682 * sync task leaves it in place 1683 */ 1684 if (f->state == I40E_FILTER_REMOVE) 1685 f->state = I40E_FILTER_ACTIVE; 1686 1687 return f; 1688} 1689 1690/** 1691 * __i40e_del_filter - Remove a specific filter from the VSI 1692 * @vsi: VSI to remove from 1693 * @f: the filter to remove from the list 1694 * 1695 * This function requires you've found * the exact filter you will remove 1696 * already, such as via i40e_find_filter or i40e_find_mac. 1697 * 1698 * NOTE: This function is expected to be called with mac_filter_hash_lock 1699 * being held. 1700 * ANOTHER NOTE: This function MUST be called from within the context of 1701 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1702 * instead of list_for_each_entry(). 1703 **/ 1704void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1705{ 1706 if (!f) 1707 return; 1708 1709 /* If the filter was never added to firmware then we can just delete it 1710 * directly and we don't want to set the status to remove or else an 1711 * admin queue command will unnecessarily fire. 1712 */ 1713 if ((f->state == I40E_FILTER_FAILED) || 1714 (f->state == I40E_FILTER_NEW)) { 1715 hash_del(&f->hlist); 1716 kfree(f); 1717 } else { 1718 f->state = I40E_FILTER_REMOVE; 1719 } 1720 1721 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1722 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1723} 1724 1725/** 1726 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1727 * @vsi: the VSI to be searched 1728 * @macaddr: the mac address to be filtered 1729 * 1730 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1731 * go through all the macvlan filters and add a macvlan filter for each 1732 * unique vlan that already exists. If a PVID has been assigned, instead only 1733 * add the macaddr to that VLAN. 1734 * 1735 * Returns last filter added on success, else NULL 1736 **/ 1737struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1738 const u8 *macaddr) 1739{ 1740 struct i40e_mac_filter *f, *add = NULL; 1741 struct hlist_node *h; 1742 int bkt; 1743 1744 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1745 if (vsi->info.pvid) 1746 return i40e_add_filter(vsi, macaddr, 1747 le16_to_cpu(vsi->info.pvid)); 1748 1749 if (!i40e_is_vsi_in_vlan(vsi)) 1750 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1751 1752 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1753 if (f->state == I40E_FILTER_REMOVE) 1754 continue; 1755 add = i40e_add_filter(vsi, macaddr, f->vlan); 1756 if (!add) 1757 return NULL; 1758 } 1759 1760 return add; 1761} 1762 1763/** 1764 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1765 * @vsi: the VSI to be searched 1766 * @macaddr: the mac address to be removed 1767 * 1768 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1769 * associated with. 1770 * 1771 * Returns 0 for success, or error 1772 **/ 1773int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1774{ 1775 struct i40e_mac_filter *f; 1776 struct hlist_node *h; 1777 bool found = false; 1778 int bkt; 1779 1780 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1781 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1782 if (ether_addr_equal(macaddr, f->macaddr)) { 1783 __i40e_del_filter(vsi, f); 1784 found = true; 1785 } 1786 } 1787 1788 if (found) 1789 return 0; 1790 else 1791 return -ENOENT; 1792} 1793 1794/** 1795 * i40e_set_mac - NDO callback to set mac address 1796 * @netdev: network interface device structure 1797 * @p: pointer to an address structure 1798 * 1799 * Returns 0 on success, negative on failure 1800 **/ 1801static int i40e_set_mac(struct net_device *netdev, void *p) 1802{ 1803 struct i40e_netdev_priv *np = netdev_priv(netdev); 1804 struct i40e_vsi *vsi = np->vsi; 1805 struct i40e_pf *pf = vsi->back; 1806 struct i40e_hw *hw = &pf->hw; 1807 struct sockaddr *addr = p; 1808 1809 if (!is_valid_ether_addr(addr->sa_data)) 1810 return -EADDRNOTAVAIL; 1811 1812 if (test_bit(__I40E_DOWN, pf->state) || 1813 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1814 return -EADDRNOTAVAIL; 1815 1816 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1817 netdev_info(netdev, "returning to hw mac address %pM\n", 1818 hw->mac.addr); 1819 else 1820 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1821 1822 /* Copy the address first, so that we avoid a possible race with 1823 * .set_rx_mode(). 1824 * - Remove old address from MAC filter 1825 * - Copy new address 1826 * - Add new address to MAC filter 1827 */ 1828 spin_lock_bh(&vsi->mac_filter_hash_lock); 1829 i40e_del_mac_filter(vsi, netdev->dev_addr); 1830 eth_hw_addr_set(netdev, addr->sa_data); 1831 i40e_add_mac_filter(vsi, netdev->dev_addr); 1832 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1833 1834 if (vsi->type == I40E_VSI_MAIN) { 1835 int ret; 1836 1837 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1838 addr->sa_data, NULL); 1839 if (ret) 1840 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n", 1841 ERR_PTR(ret), 1842 libie_aq_str(hw->aq.asq_last_status)); 1843 } 1844 1845 /* schedule our worker thread which will take care of 1846 * applying the new filter changes 1847 */ 1848 i40e_service_event_schedule(pf); 1849 return 0; 1850} 1851 1852/** 1853 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1854 * @vsi: vsi structure 1855 * @seed: RSS hash seed 1856 * @lut: pointer to lookup table of lut_size 1857 * @lut_size: size of the lookup table 1858 **/ 1859static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1860 u8 *lut, u16 lut_size) 1861{ 1862 struct i40e_pf *pf = vsi->back; 1863 struct i40e_hw *hw = &pf->hw; 1864 int ret = 0; 1865 1866 if (seed) { 1867 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1868 (struct i40e_aqc_get_set_rss_key_data *)seed; 1869 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1870 if (ret) { 1871 dev_info(&pf->pdev->dev, 1872 "Cannot set RSS key, err %pe aq_err %s\n", 1873 ERR_PTR(ret), 1874 libie_aq_str(hw->aq.asq_last_status)); 1875 return ret; 1876 } 1877 } 1878 if (lut) { 1879 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1880 1881 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1882 if (ret) { 1883 dev_info(&pf->pdev->dev, 1884 "Cannot set RSS lut, err %pe aq_err %s\n", 1885 ERR_PTR(ret), 1886 libie_aq_str(hw->aq.asq_last_status)); 1887 return ret; 1888 } 1889 } 1890 return ret; 1891} 1892 1893/** 1894 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1895 * @vsi: VSI structure 1896 **/ 1897static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1898{ 1899 struct i40e_pf *pf = vsi->back; 1900 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1901 u8 *lut; 1902 int ret; 1903 1904 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 1905 return 0; 1906 if (!vsi->rss_size) 1907 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1908 vsi->num_queue_pairs); 1909 if (!vsi->rss_size) 1910 return -EINVAL; 1911 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1912 if (!lut) 1913 return -ENOMEM; 1914 1915 /* Use the user configured hash keys and lookup table if there is one, 1916 * otherwise use default 1917 */ 1918 if (vsi->rss_lut_user) 1919 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1920 else 1921 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1922 if (vsi->rss_hkey_user) 1923 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1924 else 1925 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1926 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1927 kfree(lut); 1928 return ret; 1929} 1930 1931/** 1932 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1933 * @vsi: the VSI being configured, 1934 * @ctxt: VSI context structure 1935 * @enabled_tc: number of traffic classes to enable 1936 * 1937 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1938 **/ 1939static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1940 struct i40e_vsi_context *ctxt, 1941 u8 enabled_tc) 1942{ 1943 u16 qcount = 0, max_qcount, qmap, sections = 0; 1944 int i, override_q, pow, num_qps, ret; 1945 u8 netdev_tc = 0, offset = 0; 1946 1947 if (vsi->type != I40E_VSI_MAIN) 1948 return -EINVAL; 1949 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1950 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1951 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1952 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1953 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1954 1955 /* find the next higher power-of-2 of num queue pairs */ 1956 pow = ilog2(num_qps); 1957 if (!is_power_of_2(num_qps)) 1958 pow++; 1959 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1960 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1961 1962 /* Setup queue offset/count for all TCs for given VSI */ 1963 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1964 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1965 /* See if the given TC is enabled for the given VSI */ 1966 if (vsi->tc_config.enabled_tc & BIT(i)) { 1967 offset = vsi->mqprio_qopt.qopt.offset[i]; 1968 qcount = vsi->mqprio_qopt.qopt.count[i]; 1969 if (qcount > max_qcount) 1970 max_qcount = qcount; 1971 vsi->tc_config.tc_info[i].qoffset = offset; 1972 vsi->tc_config.tc_info[i].qcount = qcount; 1973 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1974 } else { 1975 /* TC is not enabled so set the offset to 1976 * default queue and allocate one queue 1977 * for the given TC. 1978 */ 1979 vsi->tc_config.tc_info[i].qoffset = 0; 1980 vsi->tc_config.tc_info[i].qcount = 1; 1981 vsi->tc_config.tc_info[i].netdev_tc = 0; 1982 } 1983 } 1984 1985 /* Set actual Tx/Rx queue pairs */ 1986 vsi->num_queue_pairs = offset + qcount; 1987 1988 /* Setup queue TC[0].qmap for given VSI context */ 1989 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1990 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1991 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1992 ctxt->info.valid_sections |= cpu_to_le16(sections); 1993 1994 /* Reconfigure RSS for main VSI with max queue count */ 1995 vsi->rss_size = max_qcount; 1996 ret = i40e_vsi_config_rss(vsi); 1997 if (ret) { 1998 dev_info(&vsi->back->pdev->dev, 1999 "Failed to reconfig rss for num_queues (%u)\n", 2000 max_qcount); 2001 return ret; 2002 } 2003 vsi->reconfig_rss = true; 2004 dev_dbg(&vsi->back->pdev->dev, 2005 "Reconfigured rss with num_queues (%u)\n", max_qcount); 2006 2007 /* Find queue count available for channel VSIs and starting offset 2008 * for channel VSIs 2009 */ 2010 override_q = vsi->mqprio_qopt.qopt.count[0]; 2011 if (override_q && override_q < vsi->num_queue_pairs) { 2012 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 2013 vsi->next_base_queue = override_q; 2014 } 2015 return 0; 2016} 2017 2018/** 2019 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 2020 * @vsi: the VSI being setup 2021 * @ctxt: VSI context structure 2022 * @enabled_tc: Enabled TCs bitmap 2023 * @is_add: True if called before Add VSI 2024 * 2025 * Setup VSI queue mapping for enabled traffic classes. 2026 **/ 2027static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 2028 struct i40e_vsi_context *ctxt, 2029 u8 enabled_tc, 2030 bool is_add) 2031{ 2032 struct i40e_pf *pf = vsi->back; 2033 u16 num_tc_qps = 0; 2034 u16 sections = 0; 2035 u8 netdev_tc = 0; 2036 u16 numtc = 1; 2037 u16 qcount; 2038 u8 offset; 2039 u16 qmap; 2040 int i; 2041 2042 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2043 offset = 0; 2044 /* zero out queue mapping, it will get updated on the end of the function */ 2045 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping)); 2046 2047 if (vsi->type == I40E_VSI_MAIN) { 2048 /* This code helps add more queue to the VSI if we have 2049 * more cores than RSS can support, the higher cores will 2050 * be served by ATR or other filters. Furthermore, the 2051 * non-zero req_queue_pairs says that user requested a new 2052 * queue count via ethtool's set_channels, so use this 2053 * value for queues distribution across traffic classes 2054 * We need at least one queue pair for the interface 2055 * to be usable as we see in else statement. 2056 */ 2057 if (vsi->req_queue_pairs > 0) 2058 vsi->num_queue_pairs = vsi->req_queue_pairs; 2059 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2060 vsi->num_queue_pairs = pf->num_lan_msix; 2061 else 2062 vsi->num_queue_pairs = 1; 2063 } 2064 2065 /* Number of queues per enabled TC */ 2066 if (vsi->type == I40E_VSI_MAIN || 2067 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0)) 2068 num_tc_qps = vsi->num_queue_pairs; 2069 else 2070 num_tc_qps = vsi->alloc_queue_pairs; 2071 2072 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 2073 /* Find numtc from enabled TC bitmap */ 2074 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2075 if (enabled_tc & BIT(i)) /* TC is enabled */ 2076 numtc++; 2077 } 2078 if (!numtc) { 2079 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 2080 numtc = 1; 2081 } 2082 num_tc_qps = num_tc_qps / numtc; 2083 num_tc_qps = min_t(int, num_tc_qps, 2084 i40e_pf_get_max_q_per_tc(pf)); 2085 } 2086 2087 vsi->tc_config.numtc = numtc; 2088 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 2089 2090 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 2091 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 2092 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 2093 2094 /* Setup queue offset/count for all TCs for given VSI */ 2095 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2096 /* See if the given TC is enabled for the given VSI */ 2097 if (vsi->tc_config.enabled_tc & BIT(i)) { 2098 /* TC is enabled */ 2099 int pow, num_qps; 2100 2101 switch (vsi->type) { 2102 case I40E_VSI_MAIN: 2103 if ((!test_bit(I40E_FLAG_FD_SB_ENA, 2104 pf->flags) && 2105 !test_bit(I40E_FLAG_FD_ATR_ENA, 2106 pf->flags)) || 2107 vsi->tc_config.enabled_tc != 1) { 2108 qcount = min_t(int, pf->alloc_rss_size, 2109 num_tc_qps); 2110 break; 2111 } 2112 fallthrough; 2113 case I40E_VSI_FDIR: 2114 case I40E_VSI_SRIOV: 2115 case I40E_VSI_VMDQ2: 2116 default: 2117 qcount = num_tc_qps; 2118 WARN_ON(i != 0); 2119 break; 2120 } 2121 vsi->tc_config.tc_info[i].qoffset = offset; 2122 vsi->tc_config.tc_info[i].qcount = qcount; 2123 2124 /* find the next higher power-of-2 of num queue pairs */ 2125 num_qps = qcount; 2126 pow = 0; 2127 while (num_qps && (BIT_ULL(pow) < qcount)) { 2128 pow++; 2129 num_qps >>= 1; 2130 } 2131 2132 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 2133 qmap = 2134 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 2135 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 2136 2137 offset += qcount; 2138 } else { 2139 /* TC is not enabled so set the offset to 2140 * default queue and allocate one queue 2141 * for the given TC. 2142 */ 2143 vsi->tc_config.tc_info[i].qoffset = 0; 2144 vsi->tc_config.tc_info[i].qcount = 1; 2145 vsi->tc_config.tc_info[i].netdev_tc = 0; 2146 2147 qmap = 0; 2148 } 2149 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 2150 } 2151 /* Do not change previously set num_queue_pairs for PFs and VFs*/ 2152 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) || 2153 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) || 2154 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV)) 2155 vsi->num_queue_pairs = offset; 2156 2157 /* Scheduler section valid can only be set for ADD VSI */ 2158 if (is_add) { 2159 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 2160 2161 ctxt->info.up_enable_bits = enabled_tc; 2162 } 2163 if (vsi->type == I40E_VSI_SRIOV) { 2164 ctxt->info.mapping_flags |= 2165 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 2166 for (i = 0; i < vsi->num_queue_pairs; i++) 2167 ctxt->info.queue_mapping[i] = 2168 cpu_to_le16(vsi->base_queue + i); 2169 } else { 2170 ctxt->info.mapping_flags |= 2171 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2172 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 2173 } 2174 ctxt->info.valid_sections |= cpu_to_le16(sections); 2175} 2176 2177/** 2178 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 2179 * @netdev: the netdevice 2180 * @addr: address to add 2181 * 2182 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 2183 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2184 */ 2185static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 2186{ 2187 struct i40e_netdev_priv *np = netdev_priv(netdev); 2188 struct i40e_vsi *vsi = np->vsi; 2189 2190 if (i40e_add_mac_filter(vsi, addr)) 2191 return 0; 2192 else 2193 return -ENOMEM; 2194} 2195 2196/** 2197 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 2198 * @netdev: the netdevice 2199 * @addr: address to add 2200 * 2201 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 2202 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2203 */ 2204static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 2205{ 2206 struct i40e_netdev_priv *np = netdev_priv(netdev); 2207 struct i40e_vsi *vsi = np->vsi; 2208 2209 /* Under some circumstances, we might receive a request to delete 2210 * our own device address from our uc list. Because we store the 2211 * device address in the VSI's MAC/VLAN filter list, we need to ignore 2212 * such requests and not delete our device address from this list. 2213 */ 2214 if (ether_addr_equal(addr, netdev->dev_addr)) 2215 return 0; 2216 2217 i40e_del_mac_filter(vsi, addr); 2218 2219 return 0; 2220} 2221 2222/** 2223 * i40e_set_rx_mode - NDO callback to set the netdev filters 2224 * @netdev: network interface device structure 2225 **/ 2226static void i40e_set_rx_mode(struct net_device *netdev) 2227{ 2228 struct i40e_netdev_priv *np = netdev_priv(netdev); 2229 struct i40e_vsi *vsi = np->vsi; 2230 2231 spin_lock_bh(&vsi->mac_filter_hash_lock); 2232 2233 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2234 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2235 2236 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2237 2238 /* check for other flag changes */ 2239 if (vsi->current_netdev_flags != vsi->netdev->flags) { 2240 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2241 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 2242 } 2243 i40e_service_event_schedule(vsi->back); 2244} 2245 2246/** 2247 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 2248 * @vsi: Pointer to VSI struct 2249 * @from: Pointer to list which contains MAC filter entries - changes to 2250 * those entries needs to be undone. 2251 * 2252 * MAC filter entries from this list were slated for deletion. 2253 **/ 2254static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 2255 struct hlist_head *from) 2256{ 2257 struct i40e_mac_filter *f; 2258 struct hlist_node *h; 2259 2260 hlist_for_each_entry_safe(f, h, from, hlist) { 2261 u64 key = i40e_addr_to_hkey(f->macaddr); 2262 2263 /* Move the element back into MAC filter list*/ 2264 hlist_del(&f->hlist); 2265 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2266 } 2267} 2268 2269/** 2270 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2271 * @vsi: Pointer to vsi struct 2272 * @from: Pointer to list which contains MAC filter entries - changes to 2273 * those entries needs to be undone. 2274 * 2275 * MAC filter entries from this list were slated for addition. 2276 **/ 2277static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2278 struct hlist_head *from) 2279{ 2280 struct i40e_new_mac_filter *new; 2281 struct hlist_node *h; 2282 2283 hlist_for_each_entry_safe(new, h, from, hlist) { 2284 /* We can simply free the wrapper structure */ 2285 hlist_del(&new->hlist); 2286 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2287 kfree(new); 2288 } 2289} 2290 2291/** 2292 * i40e_next_filter - Get the next non-broadcast filter from a list 2293 * @next: pointer to filter in list 2294 * 2295 * Returns the next non-broadcast filter in the list. Required so that we 2296 * ignore broadcast filters within the list, since these are not handled via 2297 * the normal firmware update path. 2298 */ 2299static 2300struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2301{ 2302 hlist_for_each_entry_continue(next, hlist) { 2303 if (!is_broadcast_ether_addr(next->f->macaddr)) 2304 return next; 2305 } 2306 2307 return NULL; 2308} 2309 2310/** 2311 * i40e_update_filter_state - Update filter state based on return data 2312 * from firmware 2313 * @count: Number of filters added 2314 * @add_list: return data from fw 2315 * @add_head: pointer to first filter in current batch 2316 * 2317 * MAC filter entries from list were slated to be added to device. Returns 2318 * number of successful filters. Note that 0 does NOT mean success! 2319 **/ 2320static int 2321i40e_update_filter_state(int count, 2322 struct i40e_aqc_add_macvlan_element_data *add_list, 2323 struct i40e_new_mac_filter *add_head) 2324{ 2325 int retval = 0; 2326 int i; 2327 2328 for (i = 0; i < count; i++) { 2329 /* Always check status of each filter. We don't need to check 2330 * the firmware return status because we pre-set the filter 2331 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2332 * request to the adminq. Thus, if it no longer matches then 2333 * we know the filter is active. 2334 */ 2335 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2336 add_head->state = I40E_FILTER_FAILED; 2337 } else { 2338 add_head->state = I40E_FILTER_ACTIVE; 2339 retval++; 2340 } 2341 2342 add_head = i40e_next_filter(add_head); 2343 if (!add_head) 2344 break; 2345 } 2346 2347 return retval; 2348} 2349 2350/** 2351 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2352 * @vsi: ptr to the VSI 2353 * @vsi_name: name to display in messages 2354 * @list: the list of filters to send to firmware 2355 * @num_del: the number of filters to delete 2356 * @retval: Set to -EIO on failure to delete 2357 * 2358 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2359 * *retval instead of a return value so that success does not force ret_val to 2360 * be set to 0. This ensures that a sequence of calls to this function 2361 * preserve the previous value of *retval on successful delete. 2362 */ 2363static 2364void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2365 struct i40e_aqc_remove_macvlan_element_data *list, 2366 int num_del, int *retval) 2367{ 2368 struct i40e_hw *hw = &vsi->back->hw; 2369 enum libie_aq_err aq_status; 2370 int aq_ret; 2371 2372 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL, 2373 &aq_status); 2374 2375 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2376 if (aq_ret && !(aq_status == LIBIE_AQ_RC_ENOENT)) { 2377 *retval = -EIO; 2378 dev_info(&vsi->back->pdev->dev, 2379 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n", 2380 vsi_name, ERR_PTR(aq_ret), libie_aq_str(aq_status)); 2381 } 2382} 2383 2384/** 2385 * i40e_aqc_add_filters - Request firmware to add a set of filters 2386 * @vsi: ptr to the VSI 2387 * @vsi_name: name to display in messages 2388 * @list: the list of filters to send to firmware 2389 * @add_head: Position in the add hlist 2390 * @num_add: the number of filters to add 2391 * 2392 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2393 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2394 * space for more filters. 2395 */ 2396static 2397void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2398 struct i40e_aqc_add_macvlan_element_data *list, 2399 struct i40e_new_mac_filter *add_head, 2400 int num_add) 2401{ 2402 struct i40e_hw *hw = &vsi->back->hw; 2403 enum libie_aq_err aq_status; 2404 int fcnt; 2405 2406 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status); 2407 fcnt = i40e_update_filter_state(num_add, list, add_head); 2408 2409 if (fcnt != num_add) { 2410 if (vsi->type == I40E_VSI_MAIN) { 2411 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2412 dev_warn(&vsi->back->pdev->dev, 2413 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2414 libie_aq_str(aq_status), vsi_name); 2415 } else if (vsi->type == I40E_VSI_SRIOV || 2416 vsi->type == I40E_VSI_VMDQ1 || 2417 vsi->type == I40E_VSI_VMDQ2) { 2418 dev_warn(&vsi->back->pdev->dev, 2419 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2420 libie_aq_str(aq_status), vsi_name, vsi_name); 2421 } else { 2422 dev_warn(&vsi->back->pdev->dev, 2423 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2424 libie_aq_str(aq_status), vsi_name, vsi->type); 2425 } 2426 } 2427} 2428 2429/** 2430 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2431 * @vsi: pointer to the VSI 2432 * @vsi_name: the VSI name 2433 * @f: filter data 2434 * 2435 * This function sets or clears the promiscuous broadcast flags for VLAN 2436 * filters in order to properly receive broadcast frames. Assumes that only 2437 * broadcast filters are passed. 2438 * 2439 * Returns status indicating success or failure; 2440 **/ 2441static int 2442i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2443 struct i40e_mac_filter *f) 2444{ 2445 bool enable = f->state == I40E_FILTER_NEW || 2446 f->state == I40E_FILTER_NEW_SYNC; 2447 struct i40e_hw *hw = &vsi->back->hw; 2448 int aq_ret; 2449 2450 if (f->vlan == I40E_VLAN_ANY) { 2451 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2452 vsi->seid, 2453 enable, 2454 NULL); 2455 } else { 2456 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2457 vsi->seid, 2458 enable, 2459 f->vlan, 2460 NULL); 2461 } 2462 2463 if (aq_ret) { 2464 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2465 dev_warn(&vsi->back->pdev->dev, 2466 "Error %s, forcing overflow promiscuous on %s\n", 2467 libie_aq_str(hw->aq.asq_last_status), vsi_name); 2468 } 2469 2470 return aq_ret; 2471} 2472 2473/** 2474 * i40e_set_promiscuous - set promiscuous mode 2475 * @pf: board private structure 2476 * @promisc: promisc on or off 2477 * 2478 * There are different ways of setting promiscuous mode on a PF depending on 2479 * what state/environment we're in. This identifies and sets it appropriately. 2480 * Returns 0 on success. 2481 **/ 2482static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2483{ 2484 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 2485 struct i40e_hw *hw = &pf->hw; 2486 int aq_ret; 2487 2488 if (vsi->type == I40E_VSI_MAIN && 2489 i40e_pf_get_main_veb(pf) && 2490 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 2491 /* set defport ON for Main VSI instead of true promisc 2492 * this way we will get all unicast/multicast and VLAN 2493 * promisc behavior but will not get VF or VMDq traffic 2494 * replicated on the Main VSI. 2495 */ 2496 if (promisc) 2497 aq_ret = i40e_aq_set_default_vsi(hw, 2498 vsi->seid, 2499 NULL); 2500 else 2501 aq_ret = i40e_aq_clear_default_vsi(hw, 2502 vsi->seid, 2503 NULL); 2504 if (aq_ret) { 2505 dev_info(&pf->pdev->dev, 2506 "Set default VSI failed, err %pe, aq_err %s\n", 2507 ERR_PTR(aq_ret), 2508 libie_aq_str(hw->aq.asq_last_status)); 2509 } 2510 } else { 2511 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2512 hw, 2513 vsi->seid, 2514 promisc, NULL, 2515 true); 2516 if (aq_ret) { 2517 dev_info(&pf->pdev->dev, 2518 "set unicast promisc failed, err %pe, aq_err %s\n", 2519 ERR_PTR(aq_ret), 2520 libie_aq_str(hw->aq.asq_last_status)); 2521 } 2522 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2523 hw, 2524 vsi->seid, 2525 promisc, NULL); 2526 if (aq_ret) { 2527 dev_info(&pf->pdev->dev, 2528 "set multicast promisc failed, err %pe, aq_err %s\n", 2529 ERR_PTR(aq_ret), 2530 libie_aq_str(hw->aq.asq_last_status)); 2531 } 2532 } 2533 2534 if (!aq_ret) 2535 pf->cur_promisc = promisc; 2536 2537 return aq_ret; 2538} 2539 2540/** 2541 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2542 * @vsi: ptr to the VSI 2543 * 2544 * Push any outstanding VSI filter changes through the AdminQ. 2545 * 2546 * Returns 0 or error value 2547 **/ 2548int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2549{ 2550 struct hlist_head tmp_add_list, tmp_del_list; 2551 struct i40e_mac_filter *f; 2552 struct i40e_new_mac_filter *new, *add_head = NULL; 2553 struct i40e_hw *hw = &vsi->back->hw; 2554 bool old_overflow, new_overflow; 2555 unsigned int failed_filters = 0; 2556 unsigned int vlan_filters = 0; 2557 char vsi_name[16] = "PF"; 2558 int filter_list_len = 0; 2559 u32 changed_flags = 0; 2560 struct hlist_node *h; 2561 struct i40e_pf *pf; 2562 int num_add = 0; 2563 int num_del = 0; 2564 int aq_ret = 0; 2565 int retval = 0; 2566 u16 cmd_flags; 2567 int list_size; 2568 int bkt; 2569 2570 /* empty array typed pointers, kcalloc later */ 2571 struct i40e_aqc_add_macvlan_element_data *add_list; 2572 struct i40e_aqc_remove_macvlan_element_data *del_list; 2573 2574 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2575 usleep_range(1000, 2000); 2576 pf = vsi->back; 2577 2578 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2579 2580 if (vsi->netdev) { 2581 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2582 vsi->current_netdev_flags = vsi->netdev->flags; 2583 } 2584 2585 INIT_HLIST_HEAD(&tmp_add_list); 2586 INIT_HLIST_HEAD(&tmp_del_list); 2587 2588 if (vsi->type == I40E_VSI_SRIOV) 2589 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2590 else if (vsi->type != I40E_VSI_MAIN) 2591 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2592 2593 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2594 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2595 2596 spin_lock_bh(&vsi->mac_filter_hash_lock); 2597 /* Create a list of filters to delete. */ 2598 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2599 if (f->state == I40E_FILTER_REMOVE) { 2600 /* Move the element into temporary del_list */ 2601 hash_del(&f->hlist); 2602 hlist_add_head(&f->hlist, &tmp_del_list); 2603 2604 /* Avoid counting removed filters */ 2605 continue; 2606 } 2607 if (f->state == I40E_FILTER_NEW) { 2608 /* Create a temporary i40e_new_mac_filter */ 2609 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2610 if (!new) 2611 goto err_no_memory_locked; 2612 2613 /* Store pointer to the real filter */ 2614 new->f = f; 2615 new->state = f->state; 2616 2617 /* Add it to the hash list */ 2618 hlist_add_head(&new->hlist, &tmp_add_list); 2619 f->state = I40E_FILTER_NEW_SYNC; 2620 } 2621 2622 /* Count the number of active (current and new) VLAN 2623 * filters we have now. Does not count filters which 2624 * are marked for deletion. 2625 */ 2626 if (f->vlan > 0) 2627 vlan_filters++; 2628 } 2629 2630 if (vsi->type != I40E_VSI_SRIOV) 2631 retval = i40e_correct_mac_vlan_filters 2632 (vsi, &tmp_add_list, &tmp_del_list, 2633 vlan_filters); 2634 else if (pf->vf) 2635 retval = i40e_correct_vf_mac_vlan_filters 2636 (vsi, &tmp_add_list, &tmp_del_list, 2637 vlan_filters, pf->vf[vsi->vf_id].trusted); 2638 2639 hlist_for_each_entry(new, &tmp_add_list, hlist) 2640 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1); 2641 2642 if (retval) 2643 goto err_no_memory_locked; 2644 2645 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2646 } 2647 2648 /* Now process 'del_list' outside the lock */ 2649 if (!hlist_empty(&tmp_del_list)) { 2650 filter_list_len = hw->aq.asq_buf_size / 2651 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2652 list_size = filter_list_len * 2653 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2654 del_list = kzalloc(list_size, GFP_ATOMIC); 2655 if (!del_list) 2656 goto err_no_memory; 2657 2658 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2659 cmd_flags = 0; 2660 2661 /* handle broadcast filters by updating the broadcast 2662 * promiscuous flag and release filter list. 2663 */ 2664 if (is_broadcast_ether_addr(f->macaddr)) { 2665 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2666 2667 hlist_del(&f->hlist); 2668 kfree(f); 2669 continue; 2670 } 2671 2672 /* add to delete list */ 2673 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2674 if (f->vlan == I40E_VLAN_ANY) { 2675 del_list[num_del].vlan_tag = 0; 2676 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2677 } else { 2678 del_list[num_del].vlan_tag = 2679 cpu_to_le16((u16)(f->vlan)); 2680 } 2681 2682 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2683 del_list[num_del].flags = cmd_flags; 2684 num_del++; 2685 2686 /* flush a full buffer */ 2687 if (num_del == filter_list_len) { 2688 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2689 num_del, &retval); 2690 memset(del_list, 0, list_size); 2691 num_del = 0; 2692 } 2693 /* Release memory for MAC filter entries which were 2694 * synced up with HW. 2695 */ 2696 hlist_del(&f->hlist); 2697 kfree(f); 2698 } 2699 2700 if (num_del) { 2701 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2702 num_del, &retval); 2703 } 2704 2705 kfree(del_list); 2706 del_list = NULL; 2707 } 2708 2709 if (!hlist_empty(&tmp_add_list)) { 2710 /* Do all the adds now. */ 2711 filter_list_len = hw->aq.asq_buf_size / 2712 sizeof(struct i40e_aqc_add_macvlan_element_data); 2713 list_size = filter_list_len * 2714 sizeof(struct i40e_aqc_add_macvlan_element_data); 2715 add_list = kzalloc(list_size, GFP_ATOMIC); 2716 if (!add_list) 2717 goto err_no_memory; 2718 2719 num_add = 0; 2720 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2721 /* handle broadcast filters by updating the broadcast 2722 * promiscuous flag instead of adding a MAC filter. 2723 */ 2724 if (is_broadcast_ether_addr(new->f->macaddr)) { 2725 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2726 new->f)) 2727 new->state = I40E_FILTER_FAILED; 2728 else 2729 new->state = I40E_FILTER_ACTIVE; 2730 continue; 2731 } 2732 2733 /* add to add array */ 2734 if (num_add == 0) 2735 add_head = new; 2736 cmd_flags = 0; 2737 ether_addr_copy(add_list[num_add].mac_addr, 2738 new->f->macaddr); 2739 if (new->f->vlan == I40E_VLAN_ANY) { 2740 add_list[num_add].vlan_tag = 0; 2741 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2742 } else { 2743 add_list[num_add].vlan_tag = 2744 cpu_to_le16((u16)(new->f->vlan)); 2745 } 2746 add_list[num_add].queue_number = 0; 2747 /* set invalid match method for later detection */ 2748 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2749 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2750 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2751 num_add++; 2752 2753 /* flush a full buffer */ 2754 if (num_add == filter_list_len) { 2755 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2756 add_head, num_add); 2757 memset(add_list, 0, list_size); 2758 num_add = 0; 2759 } 2760 } 2761 if (num_add) { 2762 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2763 num_add); 2764 } 2765 /* Now move all of the filters from the temp add list back to 2766 * the VSI's list. 2767 */ 2768 spin_lock_bh(&vsi->mac_filter_hash_lock); 2769 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2770 /* Only update the state if we're still NEW */ 2771 if (new->f->state == I40E_FILTER_NEW || 2772 new->f->state == I40E_FILTER_NEW_SYNC) 2773 new->f->state = new->state; 2774 hlist_del(&new->hlist); 2775 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2776 kfree(new); 2777 } 2778 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2779 kfree(add_list); 2780 add_list = NULL; 2781 } 2782 2783 /* Determine the number of active and failed filters. */ 2784 spin_lock_bh(&vsi->mac_filter_hash_lock); 2785 vsi->active_filters = 0; 2786 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2787 if (f->state == I40E_FILTER_ACTIVE) 2788 vsi->active_filters++; 2789 else if (f->state == I40E_FILTER_FAILED) 2790 failed_filters++; 2791 } 2792 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2793 2794 /* Check if we are able to exit overflow promiscuous mode. We can 2795 * safely exit if we didn't just enter, we no longer have any failed 2796 * filters, and we have reduced filters below the threshold value. 2797 */ 2798 if (old_overflow && !failed_filters && 2799 vsi->active_filters < vsi->promisc_threshold) { 2800 dev_info(&pf->pdev->dev, 2801 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2802 vsi_name); 2803 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2804 vsi->promisc_threshold = 0; 2805 } 2806 2807 /* if the VF is not trusted do not do promisc */ 2808 if (vsi->type == I40E_VSI_SRIOV && pf->vf && 2809 !pf->vf[vsi->vf_id].trusted) { 2810 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2811 goto out; 2812 } 2813 2814 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2815 2816 /* If we are entering overflow promiscuous, we need to calculate a new 2817 * threshold for when we are safe to exit 2818 */ 2819 if (!old_overflow && new_overflow) 2820 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2821 2822 /* check for changes in promiscuous modes */ 2823 if (changed_flags & IFF_ALLMULTI) { 2824 bool cur_multipromisc; 2825 2826 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2827 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2828 vsi->seid, 2829 cur_multipromisc, 2830 NULL); 2831 if (aq_ret) { 2832 retval = i40e_aq_rc_to_posix(aq_ret, 2833 hw->aq.asq_last_status); 2834 dev_info(&pf->pdev->dev, 2835 "set multi promisc failed on %s, err %pe aq_err %s\n", 2836 vsi_name, 2837 ERR_PTR(aq_ret), 2838 libie_aq_str(hw->aq.asq_last_status)); 2839 } else { 2840 dev_info(&pf->pdev->dev, "%s allmulti mode.\n", 2841 cur_multipromisc ? "entering" : "leaving"); 2842 } 2843 } 2844 2845 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2846 bool cur_promisc; 2847 2848 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2849 new_overflow); 2850 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2851 if (aq_ret) { 2852 retval = i40e_aq_rc_to_posix(aq_ret, 2853 hw->aq.asq_last_status); 2854 dev_info(&pf->pdev->dev, 2855 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n", 2856 cur_promisc ? "on" : "off", 2857 vsi_name, 2858 ERR_PTR(aq_ret), 2859 libie_aq_str(hw->aq.asq_last_status)); 2860 } 2861 } 2862out: 2863 /* if something went wrong then set the changed flag so we try again */ 2864 if (retval) 2865 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2866 2867 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2868 return retval; 2869 2870err_no_memory: 2871 /* Restore elements on the temporary add and delete lists */ 2872 spin_lock_bh(&vsi->mac_filter_hash_lock); 2873err_no_memory_locked: 2874 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2875 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2876 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2877 2878 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2879 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2880 return -ENOMEM; 2881} 2882 2883/** 2884 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2885 * @pf: board private structure 2886 **/ 2887static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2888{ 2889 struct i40e_vsi *vsi; 2890 int v; 2891 2892 if (!pf) 2893 return; 2894 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2895 return; 2896 if (test_bit(__I40E_VF_DISABLE, pf->state)) { 2897 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2898 return; 2899 } 2900 2901 i40e_pf_for_each_vsi(pf, v, vsi) { 2902 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) && 2903 !test_bit(__I40E_VSI_RELEASING, vsi->state)) { 2904 int ret = i40e_sync_vsi_filters(vsi); 2905 2906 if (ret) { 2907 /* come back and try again later */ 2908 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2909 pf->state); 2910 break; 2911 } 2912 } 2913 } 2914} 2915 2916/** 2917 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length 2918 * 2919 * @vsi: VSI to calculate rx_buf_len from 2920 */ 2921static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi) 2922{ 2923 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) 2924 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048); 2925 2926 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048; 2927} 2928 2929/** 2930 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI 2931 * @vsi: the vsi 2932 * @xdp_prog: XDP program 2933 **/ 2934static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi, 2935 struct bpf_prog *xdp_prog) 2936{ 2937 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 2938 u16 chain_len; 2939 2940 if (xdp_prog && !xdp_prog->aux->xdp_has_frags) 2941 chain_len = 1; 2942 else 2943 chain_len = I40E_MAX_CHAINED_RX_BUFFERS; 2944 2945 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER); 2946} 2947 2948/** 2949 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2950 * @netdev: network interface device structure 2951 * @new_mtu: new value for maximum frame size 2952 * 2953 * Returns 0 on success, negative on failure 2954 **/ 2955static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2956{ 2957 struct i40e_netdev_priv *np = netdev_priv(netdev); 2958 struct i40e_vsi *vsi = np->vsi; 2959 struct i40e_pf *pf = vsi->back; 2960 int frame_size; 2961 2962 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 2963 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) { 2964 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n", 2965 new_mtu, frame_size - I40E_PACKET_HDR_PAD); 2966 return -EINVAL; 2967 } 2968 2969 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2970 netdev->mtu, new_mtu); 2971 WRITE_ONCE(netdev->mtu, new_mtu); 2972 if (netif_running(netdev)) 2973 i40e_vsi_reinit_locked(vsi); 2974 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2975 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2976 return 0; 2977} 2978 2979/** 2980 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2981 * @vsi: the vsi being adjusted 2982 **/ 2983void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2984{ 2985 struct i40e_vsi_context ctxt; 2986 int ret; 2987 2988 /* Don't modify stripping options if a port VLAN is active */ 2989 if (vsi->info.pvid) 2990 return; 2991 2992 if ((vsi->info.valid_sections & 2993 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2994 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2995 return; /* already enabled */ 2996 2997 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2998 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2999 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 3000 3001 ctxt.seid = vsi->seid; 3002 ctxt.info = vsi->info; 3003 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3004 if (ret) { 3005 dev_info(&vsi->back->pdev->dev, 3006 "update vlan stripping failed, err %pe aq_err %s\n", 3007 ERR_PTR(ret), 3008 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3009 } 3010} 3011 3012/** 3013 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 3014 * @vsi: the vsi being adjusted 3015 **/ 3016void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 3017{ 3018 struct i40e_vsi_context ctxt; 3019 int ret; 3020 3021 /* Don't modify stripping options if a port VLAN is active */ 3022 if (vsi->info.pvid) 3023 return; 3024 3025 if ((vsi->info.valid_sections & 3026 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 3027 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 3028 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 3029 return; /* already disabled */ 3030 3031 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3032 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3033 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 3034 3035 ctxt.seid = vsi->seid; 3036 ctxt.info = vsi->info; 3037 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3038 if (ret) { 3039 dev_info(&vsi->back->pdev->dev, 3040 "update vlan stripping failed, err %pe aq_err %s\n", 3041 ERR_PTR(ret), 3042 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3043 } 3044} 3045 3046/** 3047 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 3048 * @vsi: the vsi being configured 3049 * @vid: vlan id to be added (0 = untagged only , -1 = any) 3050 * 3051 * This is a helper function for adding a new MAC/VLAN filter with the 3052 * specified VLAN for each existing MAC address already in the hash table. 3053 * This function does *not* perform any accounting to update filters based on 3054 * VLAN mode. 3055 * 3056 * NOTE: this function expects to be called while under the 3057 * mac_filter_hash_lock 3058 **/ 3059int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3060{ 3061 struct i40e_mac_filter *f, *add_f; 3062 struct hlist_node *h; 3063 int bkt; 3064 3065 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3066 /* If we're asked to add a filter that has been marked for 3067 * removal, it is safe to simply restore it to active state. 3068 * __i40e_del_filter will have simply deleted any filters which 3069 * were previously marked NEW or FAILED, so if it is currently 3070 * marked REMOVE it must have previously been ACTIVE. Since we 3071 * haven't yet run the sync filters task, just restore this 3072 * filter to the ACTIVE state so that the sync task leaves it 3073 * in place. 3074 */ 3075 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) { 3076 f->state = I40E_FILTER_ACTIVE; 3077 continue; 3078 } else if (f->state == I40E_FILTER_REMOVE) { 3079 continue; 3080 } 3081 add_f = i40e_add_filter(vsi, f->macaddr, vid); 3082 if (!add_f) { 3083 dev_info(&vsi->back->pdev->dev, 3084 "Could not add vlan filter %d for %pM\n", 3085 vid, f->macaddr); 3086 return -ENOMEM; 3087 } 3088 } 3089 3090 return 0; 3091} 3092 3093/** 3094 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 3095 * @vsi: the VSI being configured 3096 * @vid: VLAN id to be added 3097 **/ 3098int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 3099{ 3100 int err; 3101 3102 if (vsi->info.pvid) 3103 return -EINVAL; 3104 3105 /* The network stack will attempt to add VID=0, with the intention to 3106 * receive priority tagged packets with a VLAN of 0. Our HW receives 3107 * these packets by default when configured to receive untagged 3108 * packets, so we don't need to add a filter for this case. 3109 * Additionally, HW interprets adding a VID=0 filter as meaning to 3110 * receive *only* tagged traffic and stops receiving untagged traffic. 3111 * Thus, we do not want to actually add a filter for VID=0 3112 */ 3113 if (!vid) 3114 return 0; 3115 3116 /* Locked once because all functions invoked below iterates list*/ 3117 spin_lock_bh(&vsi->mac_filter_hash_lock); 3118 err = i40e_add_vlan_all_mac(vsi, vid); 3119 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3120 if (err) 3121 return err; 3122 3123 /* schedule our worker thread which will take care of 3124 * applying the new filter changes 3125 */ 3126 i40e_service_event_schedule(vsi->back); 3127 return 0; 3128} 3129 3130/** 3131 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 3132 * @vsi: the vsi being configured 3133 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 3134 * 3135 * This function should be used to remove all VLAN filters which match the 3136 * given VID. It does not schedule the service event and does not take the 3137 * mac_filter_hash_lock so it may be combined with other operations under 3138 * a single invocation of the mac_filter_hash_lock. 3139 * 3140 * NOTE: this function expects to be called while under the 3141 * mac_filter_hash_lock 3142 */ 3143void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3144{ 3145 struct i40e_mac_filter *f; 3146 struct hlist_node *h; 3147 int bkt; 3148 3149 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3150 if (f->vlan == vid) 3151 __i40e_del_filter(vsi, f); 3152 } 3153} 3154 3155/** 3156 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 3157 * @vsi: the VSI being configured 3158 * @vid: VLAN id to be removed 3159 **/ 3160void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 3161{ 3162 if (!vid || vsi->info.pvid) 3163 return; 3164 3165 spin_lock_bh(&vsi->mac_filter_hash_lock); 3166 i40e_rm_vlan_all_mac(vsi, vid); 3167 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3168 3169 /* schedule our worker thread which will take care of 3170 * applying the new filter changes 3171 */ 3172 i40e_service_event_schedule(vsi->back); 3173} 3174 3175/** 3176 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 3177 * @netdev: network interface to be adjusted 3178 * @proto: unused protocol value 3179 * @vid: vlan id to be added 3180 * 3181 * net_device_ops implementation for adding vlan ids 3182 **/ 3183static int i40e_vlan_rx_add_vid(struct net_device *netdev, 3184 __always_unused __be16 proto, u16 vid) 3185{ 3186 struct i40e_netdev_priv *np = netdev_priv(netdev); 3187 struct i40e_vsi *vsi = np->vsi; 3188 int ret = 0; 3189 3190 if (vid >= VLAN_N_VID) 3191 return -EINVAL; 3192 3193 ret = i40e_vsi_add_vlan(vsi, vid); 3194 if (!ret) 3195 set_bit(vid, vsi->active_vlans); 3196 3197 return ret; 3198} 3199 3200/** 3201 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 3202 * @netdev: network interface to be adjusted 3203 * @proto: unused protocol value 3204 * @vid: vlan id to be added 3205 **/ 3206static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 3207 __always_unused __be16 proto, u16 vid) 3208{ 3209 struct i40e_netdev_priv *np = netdev_priv(netdev); 3210 struct i40e_vsi *vsi = np->vsi; 3211 3212 if (vid >= VLAN_N_VID) 3213 return; 3214 set_bit(vid, vsi->active_vlans); 3215} 3216 3217/** 3218 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 3219 * @netdev: network interface to be adjusted 3220 * @proto: unused protocol value 3221 * @vid: vlan id to be removed 3222 * 3223 * net_device_ops implementation for removing vlan ids 3224 **/ 3225static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 3226 __always_unused __be16 proto, u16 vid) 3227{ 3228 struct i40e_netdev_priv *np = netdev_priv(netdev); 3229 struct i40e_vsi *vsi = np->vsi; 3230 3231 /* return code is ignored as there is nothing a user 3232 * can do about failure to remove and a log message was 3233 * already printed from the other function 3234 */ 3235 i40e_vsi_kill_vlan(vsi, vid); 3236 3237 clear_bit(vid, vsi->active_vlans); 3238 3239 return 0; 3240} 3241 3242/** 3243 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 3244 * @vsi: the vsi being brought back up 3245 **/ 3246static void i40e_restore_vlan(struct i40e_vsi *vsi) 3247{ 3248 u16 vid; 3249 3250 if (!vsi->netdev) 3251 return; 3252 3253 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3254 i40e_vlan_stripping_enable(vsi); 3255 else 3256 i40e_vlan_stripping_disable(vsi); 3257 3258 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 3259 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 3260 vid); 3261} 3262 3263/** 3264 * i40e_vsi_add_pvid - Add pvid for the VSI 3265 * @vsi: the vsi being adjusted 3266 * @vid: the vlan id to set as a PVID 3267 **/ 3268int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 3269{ 3270 struct i40e_vsi_context ctxt; 3271 int ret; 3272 3273 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3274 vsi->info.pvid = cpu_to_le16(vid); 3275 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 3276 I40E_AQ_VSI_PVLAN_INSERT_PVID | 3277 I40E_AQ_VSI_PVLAN_EMOD_STR; 3278 3279 ctxt.seid = vsi->seid; 3280 ctxt.info = vsi->info; 3281 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3282 if (ret) { 3283 dev_info(&vsi->back->pdev->dev, 3284 "add pvid failed, err %pe aq_err %s\n", 3285 ERR_PTR(ret), 3286 libie_aq_str(vsi->back->hw.aq.asq_last_status)); 3287 return -ENOENT; 3288 } 3289 3290 return 0; 3291} 3292 3293/** 3294 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3295 * @vsi: the vsi being adjusted 3296 * 3297 * Just use the vlan_rx_register() service to put it back to normal 3298 **/ 3299void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3300{ 3301 vsi->info.pvid = 0; 3302 3303 i40e_vlan_stripping_disable(vsi); 3304} 3305 3306/** 3307 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3308 * @vsi: ptr to the VSI 3309 * 3310 * If this function returns with an error, then it's possible one or 3311 * more of the rings is populated (while the rest are not). It is the 3312 * callers duty to clean those orphaned rings. 3313 * 3314 * Return 0 on success, negative on failure 3315 **/ 3316static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3317{ 3318 int i, err = 0; 3319 3320 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3321 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3322 3323 if (!i40e_enabled_xdp_vsi(vsi)) 3324 return err; 3325 3326 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3327 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3328 3329 return err; 3330} 3331 3332/** 3333 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3334 * @vsi: ptr to the VSI 3335 * 3336 * Free VSI's transmit software resources 3337 **/ 3338static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3339{ 3340 int i; 3341 3342 if (vsi->tx_rings) { 3343 for (i = 0; i < vsi->num_queue_pairs; i++) 3344 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3345 i40e_free_tx_resources(vsi->tx_rings[i]); 3346 } 3347 3348 if (vsi->xdp_rings) { 3349 for (i = 0; i < vsi->num_queue_pairs; i++) 3350 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3351 i40e_free_tx_resources(vsi->xdp_rings[i]); 3352 } 3353} 3354 3355/** 3356 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3357 * @vsi: ptr to the VSI 3358 * 3359 * If this function returns with an error, then it's possible one or 3360 * more of the rings is populated (while the rest are not). It is the 3361 * callers duty to clean those orphaned rings. 3362 * 3363 * Return 0 on success, negative on failure 3364 **/ 3365static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3366{ 3367 int i, err = 0; 3368 3369 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3370 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3371 return err; 3372} 3373 3374/** 3375 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3376 * @vsi: ptr to the VSI 3377 * 3378 * Free all receive software resources 3379 **/ 3380static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3381{ 3382 int i; 3383 3384 if (!vsi->rx_rings) 3385 return; 3386 3387 for (i = 0; i < vsi->num_queue_pairs; i++) 3388 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3389 i40e_free_rx_resources(vsi->rx_rings[i]); 3390} 3391 3392/** 3393 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3394 * @ring: The Tx ring to configure 3395 * 3396 * This enables/disables XPS for a given Tx descriptor ring 3397 * based on the TCs enabled for the VSI that ring belongs to. 3398 **/ 3399static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3400{ 3401 int cpu; 3402 3403 if (!ring->q_vector || !ring->netdev || ring->ch) 3404 return; 3405 3406 /* We only initialize XPS once, so as not to overwrite user settings */ 3407 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3408 return; 3409 3410 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3411 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3412 ring->queue_index); 3413} 3414 3415/** 3416 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled 3417 * @ring: The Tx or Rx ring 3418 * 3419 * Returns the AF_XDP buffer pool or NULL. 3420 **/ 3421static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring) 3422{ 3423 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3424 int qid = ring->queue_index; 3425 3426 if (ring_is_xdp(ring)) 3427 qid -= ring->vsi->alloc_queue_pairs; 3428 3429 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3430 return NULL; 3431 3432 return xsk_get_pool_from_qid(ring->vsi->netdev, qid); 3433} 3434 3435/** 3436 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3437 * @ring: The Tx ring to configure 3438 * 3439 * Configure the Tx descriptor ring in the HMC context. 3440 **/ 3441static int i40e_configure_tx_ring(struct i40e_ring *ring) 3442{ 3443 struct i40e_vsi *vsi = ring->vsi; 3444 u16 pf_q = vsi->base_queue + ring->queue_index; 3445 struct i40e_hw *hw = &vsi->back->hw; 3446 struct i40e_hmc_obj_txq tx_ctx; 3447 u32 qtx_ctl = 0; 3448 int err = 0; 3449 3450 if (ring_is_xdp(ring)) 3451 ring->xsk_pool = i40e_xsk_pool(ring); 3452 3453 /* some ATR related tx ring init */ 3454 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) { 3455 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 3456 ring->atr_count = 0; 3457 } else { 3458 ring->atr_sample_rate = 0; 3459 } 3460 3461 /* configure XPS */ 3462 i40e_config_xps_tx_ring(ring); 3463 3464 /* clear the context structure first */ 3465 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3466 3467 tx_ctx.new_context = 1; 3468 tx_ctx.base = (ring->dma / 128); 3469 tx_ctx.qlen = ring->count; 3470 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) || 3471 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) 3472 tx_ctx.fd_ena = 1; 3473 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags)) 3474 tx_ctx.timesync_ena = 1; 3475 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3476 if (vsi->type != I40E_VSI_FDIR) 3477 tx_ctx.head_wb_ena = 1; 3478 tx_ctx.head_wb_addr = ring->dma + 3479 (ring->count * sizeof(struct i40e_tx_desc)); 3480 3481 /* As part of VSI creation/update, FW allocates certain 3482 * Tx arbitration queue sets for each TC enabled for 3483 * the VSI. The FW returns the handles to these queue 3484 * sets as part of the response buffer to Add VSI, 3485 * Update VSI, etc. AQ commands. It is expected that 3486 * these queue set handles be associated with the Tx 3487 * queues by the driver as part of the TX queue context 3488 * initialization. This has to be done regardless of 3489 * DCB as by default everything is mapped to TC0. 3490 */ 3491 3492 if (ring->ch) 3493 tx_ctx.rdylist = 3494 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3495 3496 else 3497 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3498 3499 tx_ctx.rdylist_act = 0; 3500 3501 /* clear the context in the HMC */ 3502 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3503 if (err) { 3504 dev_info(&vsi->back->pdev->dev, 3505 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3506 ring->queue_index, pf_q, err); 3507 return -ENOMEM; 3508 } 3509 3510 /* set the context in the HMC */ 3511 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3512 if (err) { 3513 dev_info(&vsi->back->pdev->dev, 3514 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3515 ring->queue_index, pf_q, err); 3516 return -ENOMEM; 3517 } 3518 3519 /* Now associate this queue with this PCI function */ 3520 if (ring->ch) { 3521 if (ring->ch->type == I40E_VSI_VMDQ2) 3522 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3523 else 3524 return -EINVAL; 3525 3526 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3527 ring->ch->vsi_number); 3528 } else { 3529 if (vsi->type == I40E_VSI_VMDQ2) { 3530 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3531 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK, 3532 vsi->id); 3533 } else { 3534 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3535 } 3536 } 3537 3538 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id); 3539 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3540 i40e_flush(hw); 3541 3542 /* cache tail off for easier writes later */ 3543 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3544 3545 return 0; 3546} 3547 3548/** 3549 * i40e_rx_offset - Return expected offset into page to access data 3550 * @rx_ring: Ring we are requesting offset of 3551 * 3552 * Returns the offset value for ring into the data buffer. 3553 */ 3554static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 3555{ 3556 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 3557} 3558 3559/** 3560 * i40e_configure_rx_ring - Configure a receive ring context 3561 * @ring: The Rx ring to configure 3562 * 3563 * Configure the Rx descriptor ring in the HMC context. 3564 **/ 3565static int i40e_configure_rx_ring(struct i40e_ring *ring) 3566{ 3567 struct i40e_vsi *vsi = ring->vsi; 3568 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3569 u16 pf_q = vsi->base_queue + ring->queue_index; 3570 struct i40e_hw *hw = &vsi->back->hw; 3571 struct i40e_hmc_obj_rxq rx_ctx; 3572 u32 xdp_frame_sz; 3573 int err = 0; 3574 bool ok; 3575 3576 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3577 3578 /* clear the context structure first */ 3579 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3580 3581 ring->rx_buf_len = vsi->rx_buf_len; 3582 xdp_frame_sz = i40e_rx_pg_size(ring) / 2; 3583 3584 /* XDP RX-queue info only needed for RX rings exposed to XDP */ 3585 if (ring->vsi->type != I40E_VSI_MAIN) 3586 goto skip; 3587 3588 ring->xsk_pool = i40e_xsk_pool(ring); 3589 if (ring->xsk_pool) { 3590 xdp_frame_sz = xsk_pool_get_rx_frag_step(ring->xsk_pool); 3591 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); 3592 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3593 ring->queue_index, 3594 ring->q_vector->napi.napi_id, 3595 xdp_frame_sz); 3596 if (err) 3597 return err; 3598 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3599 MEM_TYPE_XSK_BUFF_POOL, 3600 NULL); 3601 if (err) 3602 goto unreg_xdp; 3603 dev_info(&vsi->back->pdev->dev, 3604 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3605 ring->queue_index); 3606 3607 } else { 3608 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, 3609 ring->queue_index, 3610 ring->q_vector->napi.napi_id, 3611 xdp_frame_sz); 3612 if (err) 3613 return err; 3614 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3615 MEM_TYPE_PAGE_SHARED, 3616 NULL); 3617 if (err) 3618 goto unreg_xdp; 3619 } 3620 3621skip: 3622 xdp_init_buff(&ring->xdp, xdp_frame_sz, &ring->xdp_rxq); 3623 3624 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3625 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3626 3627 rx_ctx.base = (ring->dma / 128); 3628 rx_ctx.qlen = ring->count; 3629 3630 /* use 16 byte descriptors */ 3631 rx_ctx.dsize = 0; 3632 3633 /* descriptor type is always zero 3634 * rx_ctx.dtype = 0; 3635 */ 3636 rx_ctx.hsplit_0 = 0; 3637 3638 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3639 if (hw->revision_id == 0) 3640 rx_ctx.lrxqthresh = 0; 3641 else 3642 rx_ctx.lrxqthresh = 1; 3643 rx_ctx.crcstrip = 1; 3644 rx_ctx.l2tsel = 1; 3645 /* this controls whether VLAN is stripped from inner headers */ 3646 rx_ctx.showiv = 0; 3647 /* set the prefena field to 1 because the manual says to */ 3648 rx_ctx.prefena = 1; 3649 3650 /* clear the context in the HMC */ 3651 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3652 if (err) { 3653 dev_info(&vsi->back->pdev->dev, 3654 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3655 ring->queue_index, pf_q, err); 3656 err = -ENOMEM; 3657 goto unreg_xdp; 3658 } 3659 3660 /* set the context in the HMC */ 3661 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3662 if (err) { 3663 dev_info(&vsi->back->pdev->dev, 3664 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3665 ring->queue_index, pf_q, err); 3666 err = -ENOMEM; 3667 goto unreg_xdp; 3668 } 3669 3670 /* configure Rx buffer alignment */ 3671 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) { 3672 if (I40E_2K_TOO_SMALL_WITH_PADDING) { 3673 dev_info(&vsi->back->pdev->dev, 3674 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); 3675 err = -EOPNOTSUPP; 3676 goto unreg_xdp; 3677 } 3678 clear_ring_build_skb_enabled(ring); 3679 } else { 3680 set_ring_build_skb_enabled(ring); 3681 } 3682 3683 ring->rx_offset = i40e_rx_offset(ring); 3684 3685 /* cache tail for quicker writes, and clear the reg before use */ 3686 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3687 writel(0, ring->tail); 3688 3689 if (ring->xsk_pool) { 3690 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 3691 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3692 } else { 3693 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3694 } 3695 if (!ok) { 3696 /* Log this in case the user has forgotten to give the kernel 3697 * any buffers, even later in the application. 3698 */ 3699 dev_info(&vsi->back->pdev->dev, 3700 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3701 ring->xsk_pool ? "AF_XDP ZC enabled " : "", 3702 ring->queue_index, pf_q); 3703 } 3704 3705 return 0; 3706unreg_xdp: 3707 if (ring->vsi->type == I40E_VSI_MAIN) 3708 xdp_rxq_info_unreg(&ring->xdp_rxq); 3709 3710 return err; 3711} 3712 3713/** 3714 * i40e_vsi_configure_tx - Configure the VSI for Tx 3715 * @vsi: VSI structure describing this set of rings and resources 3716 * 3717 * Configure the Tx VSI for operation. 3718 **/ 3719static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3720{ 3721 int err = 0; 3722 u16 i; 3723 3724 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3725 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3726 3727 if (err || !i40e_enabled_xdp_vsi(vsi)) 3728 return err; 3729 3730 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3731 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3732 3733 return err; 3734} 3735 3736/** 3737 * i40e_vsi_configure_rx - Configure the VSI for Rx 3738 * @vsi: the VSI being configured 3739 * 3740 * Configure the Rx VSI for operation. 3741 **/ 3742static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3743{ 3744 int err = 0; 3745 u16 i; 3746 3747 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 3748 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 3749 3750#if (PAGE_SIZE < 8192) 3751 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING && 3752 vsi->netdev->mtu <= ETH_DATA_LEN) { 3753 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3754 vsi->max_frame = vsi->rx_buf_len; 3755 } 3756#endif 3757 3758 /* set up individual rings */ 3759 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3760 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3761 3762 return err; 3763} 3764 3765/** 3766 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3767 * @vsi: ptr to the VSI 3768 **/ 3769static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3770{ 3771 struct i40e_ring *tx_ring, *rx_ring; 3772 u16 qoffset, qcount; 3773 int i, n; 3774 3775 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) { 3776 /* Reset the TC information */ 3777 for (i = 0; i < vsi->num_queue_pairs; i++) { 3778 rx_ring = vsi->rx_rings[i]; 3779 tx_ring = vsi->tx_rings[i]; 3780 rx_ring->dcb_tc = 0; 3781 tx_ring->dcb_tc = 0; 3782 } 3783 return; 3784 } 3785 3786 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3787 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3788 continue; 3789 3790 qoffset = vsi->tc_config.tc_info[n].qoffset; 3791 qcount = vsi->tc_config.tc_info[n].qcount; 3792 for (i = qoffset; i < (qoffset + qcount); i++) { 3793 rx_ring = vsi->rx_rings[i]; 3794 tx_ring = vsi->tx_rings[i]; 3795 rx_ring->dcb_tc = n; 3796 tx_ring->dcb_tc = n; 3797 } 3798 } 3799} 3800 3801/** 3802 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3803 * @vsi: ptr to the VSI 3804 **/ 3805static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3806{ 3807 if (vsi->netdev) 3808 i40e_set_rx_mode(vsi->netdev); 3809} 3810 3811/** 3812 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters 3813 * @pf: Pointer to the targeted PF 3814 * 3815 * Set all flow director counters to 0. 3816 */ 3817static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf) 3818{ 3819 pf->fd_tcp4_filter_cnt = 0; 3820 pf->fd_udp4_filter_cnt = 0; 3821 pf->fd_sctp4_filter_cnt = 0; 3822 pf->fd_ip4_filter_cnt = 0; 3823 pf->fd_tcp6_filter_cnt = 0; 3824 pf->fd_udp6_filter_cnt = 0; 3825 pf->fd_sctp6_filter_cnt = 0; 3826 pf->fd_ip6_filter_cnt = 0; 3827} 3828 3829/** 3830 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3831 * @vsi: Pointer to the targeted VSI 3832 * 3833 * This function replays the hlist on the hw where all the SB Flow Director 3834 * filters were saved. 3835 **/ 3836static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3837{ 3838 struct i40e_fdir_filter *filter; 3839 struct i40e_pf *pf = vsi->back; 3840 struct hlist_node *node; 3841 3842 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 3843 return; 3844 3845 /* Reset FDir counters as we're replaying all existing filters */ 3846 i40e_reset_fdir_filter_cnt(pf); 3847 3848 hlist_for_each_entry_safe(filter, node, 3849 &pf->fdir_filter_list, fdir_node) { 3850 i40e_add_del_fdir(vsi, filter, true); 3851 } 3852} 3853 3854/** 3855 * i40e_vsi_configure - Set up the VSI for action 3856 * @vsi: the VSI being configured 3857 **/ 3858static int i40e_vsi_configure(struct i40e_vsi *vsi) 3859{ 3860 int err; 3861 3862 i40e_set_vsi_rx_mode(vsi); 3863 i40e_restore_vlan(vsi); 3864 i40e_vsi_config_dcb_rings(vsi); 3865 err = i40e_vsi_configure_tx(vsi); 3866 if (!err) 3867 err = i40e_vsi_configure_rx(vsi); 3868 3869 return err; 3870} 3871 3872/** 3873 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3874 * @vsi: the VSI being configured 3875 **/ 3876static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3877{ 3878 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3879 struct i40e_pf *pf = vsi->back; 3880 struct i40e_hw *hw = &pf->hw; 3881 u16 vector; 3882 int i, q; 3883 u32 qp; 3884 3885 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3886 * and PFINT_LNKLSTn registers, e.g.: 3887 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3888 */ 3889 qp = vsi->base_queue; 3890 vector = vsi->base_vector; 3891 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3892 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3893 3894 q_vector->rx.next_update = jiffies + 1; 3895 q_vector->rx.target_itr = 3896 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3897 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3898 q_vector->rx.target_itr >> 1); 3899 q_vector->rx.current_itr = q_vector->rx.target_itr; 3900 3901 q_vector->tx.next_update = jiffies + 1; 3902 q_vector->tx.target_itr = 3903 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3904 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3905 q_vector->tx.target_itr >> 1); 3906 q_vector->tx.current_itr = q_vector->tx.target_itr; 3907 3908 /* Set ITR for software interrupts triggered after exiting 3909 * busy-loop polling. 3910 */ 3911 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1), 3912 I40E_ITR_20K); 3913 3914 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3915 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3916 3917 /* begin of linked list for RX queue assigned to this vector */ 3918 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3919 for (q = 0; q < q_vector->num_ringpairs; q++) { 3920 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3921 u32 val; 3922 3923 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3924 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3925 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3926 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3927 (I40E_QUEUE_TYPE_TX << 3928 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3929 3930 wr32(hw, I40E_QINT_RQCTL(qp), val); 3931 3932 if (has_xdp) { 3933 /* TX queue with next queue set to TX */ 3934 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3935 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3936 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3937 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3938 (I40E_QUEUE_TYPE_TX << 3939 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3940 3941 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3942 } 3943 /* TX queue with next RX or end of linked list */ 3944 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3945 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3946 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3947 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3948 (I40E_QUEUE_TYPE_RX << 3949 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3950 3951 /* Terminate the linked list */ 3952 if (q == (q_vector->num_ringpairs - 1)) 3953 val |= (I40E_QUEUE_END_OF_LIST << 3954 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3955 3956 wr32(hw, I40E_QINT_TQCTL(qp), val); 3957 qp++; 3958 } 3959 } 3960 3961 i40e_flush(hw); 3962} 3963 3964/** 3965 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3966 * @pf: pointer to private device data structure 3967 **/ 3968static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3969{ 3970 struct i40e_hw *hw = &pf->hw; 3971 u32 val; 3972 3973 /* clear things first */ 3974 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3975 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3976 3977 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3978 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3979 I40E_PFINT_ICR0_ENA_GRST_MASK | 3980 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3981 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3982 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3983 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3984 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3985 3986 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 3987 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3988 3989 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 3990 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3991 3992 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3993 3994 /* SW_ITR_IDX = 0, but don't change INTENA */ 3995 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3996 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3997 3998 /* OTHER_ITR_IDX = 0 */ 3999 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 4000} 4001 4002/** 4003 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 4004 * @vsi: the VSI being configured 4005 **/ 4006static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 4007{ 4008 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 4009 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4010 struct i40e_pf *pf = vsi->back; 4011 struct i40e_hw *hw = &pf->hw; 4012 4013 /* set the ITR configuration */ 4014 q_vector->rx.next_update = jiffies + 1; 4015 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 4016 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 4017 q_vector->rx.current_itr = q_vector->rx.target_itr; 4018 q_vector->tx.next_update = jiffies + 1; 4019 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 4020 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 4021 q_vector->tx.current_itr = q_vector->tx.target_itr; 4022 4023 i40e_enable_misc_int_causes(pf); 4024 4025 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 4026 wr32(hw, I40E_PFINT_LNKLST0, 0); 4027 4028 /* Associate the queue pair to the vector and enable the queue 4029 * interrupt RX queue in linked list with next queue set to TX 4030 */ 4031 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX)); 4032 4033 if (i40e_enabled_xdp_vsi(vsi)) { 4034 /* TX queue in linked list with next queue set to TX */ 4035 wr32(hw, I40E_QINT_TQCTL(nextqp), 4036 I40E_QINT_TQCTL_VAL(nextqp, 0, TX)); 4037 } 4038 4039 /* last TX queue so the next RX queue doesn't matter */ 4040 wr32(hw, I40E_QINT_TQCTL(0), 4041 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX)); 4042 i40e_flush(hw); 4043} 4044 4045/** 4046 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 4047 * @pf: board private structure 4048 **/ 4049void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 4050{ 4051 struct i40e_hw *hw = &pf->hw; 4052 4053 wr32(hw, I40E_PFINT_DYN_CTL0, 4054 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4055 i40e_flush(hw); 4056} 4057 4058/** 4059 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 4060 * @pf: board private structure 4061 **/ 4062void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 4063{ 4064 struct i40e_hw *hw = &pf->hw; 4065 u32 val; 4066 4067 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4068 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4069 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4070 4071 wr32(hw, I40E_PFINT_DYN_CTL0, val); 4072 i40e_flush(hw); 4073} 4074 4075/** 4076 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 4077 * @irq: interrupt number 4078 * @data: pointer to a q_vector 4079 **/ 4080static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 4081{ 4082 struct i40e_q_vector *q_vector = data; 4083 4084 if (!q_vector->tx.ring && !q_vector->rx.ring) 4085 return IRQ_HANDLED; 4086 4087 napi_schedule_irqoff(&q_vector->napi); 4088 4089 return IRQ_HANDLED; 4090} 4091 4092/** 4093 * i40e_irq_affinity_notify - Callback for affinity changes 4094 * @notify: context as to what irq was changed 4095 * @mask: the new affinity mask 4096 * 4097 * This is a callback function used by the irq_set_affinity_notifier function 4098 * so that we may register to receive changes to the irq affinity masks. 4099 **/ 4100static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 4101 const cpumask_t *mask) 4102{ 4103 struct i40e_q_vector *q_vector = 4104 container_of(notify, struct i40e_q_vector, affinity_notify); 4105 4106 cpumask_copy(&q_vector->affinity_mask, mask); 4107} 4108 4109/** 4110 * i40e_irq_affinity_release - Callback for affinity notifier release 4111 * @ref: internal core kernel usage 4112 * 4113 * This is a callback function used by the irq_set_affinity_notifier function 4114 * to inform the current notification subscriber that they will no longer 4115 * receive notifications. 4116 **/ 4117static void i40e_irq_affinity_release(struct kref *ref) {} 4118 4119/** 4120 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 4121 * @vsi: the VSI being configured 4122 * @basename: name for the vector 4123 * 4124 * Allocates MSI-X vectors and requests interrupts from the kernel. 4125 **/ 4126static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 4127{ 4128 int q_vectors = vsi->num_q_vectors; 4129 struct i40e_pf *pf = vsi->back; 4130 int base = vsi->base_vector; 4131 int rx_int_idx = 0; 4132 int tx_int_idx = 0; 4133 int vector, err; 4134 int irq_num; 4135 int cpu; 4136 4137 for (vector = 0; vector < q_vectors; vector++) { 4138 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 4139 4140 irq_num = pf->msix_entries[base + vector].vector; 4141 4142 if (q_vector->tx.ring && q_vector->rx.ring) { 4143 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4144 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 4145 tx_int_idx++; 4146 } else if (q_vector->rx.ring) { 4147 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4148 "%s-%s-%d", basename, "rx", rx_int_idx++); 4149 } else if (q_vector->tx.ring) { 4150 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4151 "%s-%s-%d", basename, "tx", tx_int_idx++); 4152 } else { 4153 /* skip this unused q_vector */ 4154 continue; 4155 } 4156 err = request_irq(irq_num, 4157 vsi->irq_handler, 4158 0, 4159 q_vector->name, 4160 q_vector); 4161 if (err) { 4162 dev_info(&pf->pdev->dev, 4163 "MSIX request_irq failed, error: %d\n", err); 4164 goto free_queue_irqs; 4165 } 4166 4167 /* register for affinity change notifications */ 4168 q_vector->irq_num = irq_num; 4169 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 4170 q_vector->affinity_notify.release = i40e_irq_affinity_release; 4171 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 4172 /* Spread affinity hints out across online CPUs. 4173 * 4174 * get_cpu_mask returns a static constant mask with 4175 * a permanent lifetime so it's ok to pass to 4176 * irq_update_affinity_hint without making a copy. 4177 */ 4178 cpu = cpumask_local_spread(q_vector->v_idx, -1); 4179 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 4180 } 4181 4182 vsi->irqs_ready = true; 4183 return 0; 4184 4185free_queue_irqs: 4186 while (vector) { 4187 vector--; 4188 irq_num = pf->msix_entries[base + vector].vector; 4189 irq_set_affinity_notifier(irq_num, NULL); 4190 irq_update_affinity_hint(irq_num, NULL); 4191 free_irq(irq_num, vsi->q_vectors[vector]); 4192 } 4193 return err; 4194} 4195 4196/** 4197 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 4198 * @vsi: the VSI being un-configured 4199 **/ 4200static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 4201{ 4202 struct i40e_pf *pf = vsi->back; 4203 struct i40e_hw *hw = &pf->hw; 4204 int base = vsi->base_vector; 4205 int i; 4206 4207 /* disable interrupt causation from each queue */ 4208 for (i = 0; i < vsi->num_queue_pairs; i++) { 4209 u32 val; 4210 4211 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 4212 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 4213 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 4214 4215 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 4216 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 4217 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 4218 4219 if (!i40e_enabled_xdp_vsi(vsi)) 4220 continue; 4221 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 4222 } 4223 4224 /* disable each interrupt */ 4225 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4226 for (i = vsi->base_vector; 4227 i < (vsi->num_q_vectors + vsi->base_vector); i++) 4228 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 4229 4230 i40e_flush(hw); 4231 for (i = 0; i < vsi->num_q_vectors; i++) 4232 synchronize_irq(pf->msix_entries[i + base].vector); 4233 } else { 4234 /* Legacy and MSI mode - this stops all interrupt handling */ 4235 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 4236 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 4237 i40e_flush(hw); 4238 synchronize_irq(pf->pdev->irq); 4239 } 4240} 4241 4242/** 4243 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 4244 * @vsi: the VSI being configured 4245 **/ 4246static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 4247{ 4248 struct i40e_pf *pf = vsi->back; 4249 int i; 4250 4251 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4252 for (i = 0; i < vsi->num_q_vectors; i++) 4253 i40e_irq_dynamic_enable(vsi, i); 4254 } else { 4255 i40e_irq_dynamic_enable_icr0(pf); 4256 } 4257 4258 i40e_flush(&pf->hw); 4259 return 0; 4260} 4261 4262/** 4263 * i40e_free_misc_vector - Free the vector that handles non-queue events 4264 * @pf: board private structure 4265 **/ 4266static void i40e_free_misc_vector(struct i40e_pf *pf) 4267{ 4268 /* Disable ICR 0 */ 4269 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 4270 i40e_flush(&pf->hw); 4271 4272 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) { 4273 free_irq(pf->msix_entries[0].vector, pf); 4274 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 4275 } 4276} 4277 4278/** 4279 * i40e_intr - MSI/Legacy and non-queue interrupt handler 4280 * @irq: interrupt number 4281 * @data: pointer to a q_vector 4282 * 4283 * This is the handler used for all MSI/Legacy interrupts, and deals 4284 * with both queue and non-queue interrupts. This is also used in 4285 * MSIX mode to handle the non-queue interrupts. 4286 **/ 4287static irqreturn_t i40e_intr(int irq, void *data) 4288{ 4289 struct i40e_pf *pf = (struct i40e_pf *)data; 4290 struct i40e_hw *hw = &pf->hw; 4291 irqreturn_t ret = IRQ_NONE; 4292 u32 icr0, icr0_remaining; 4293 u32 val, ena_mask; 4294 4295 icr0 = rd32(hw, I40E_PFINT_ICR0); 4296 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 4297 4298 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 4299 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 4300 goto enable_intr; 4301 4302 /* if interrupt but no bits showing, must be SWINT */ 4303 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 4304 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 4305 pf->sw_int_count++; 4306 4307 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 4308 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 4309 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 4310 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 4311 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 4312 } 4313 4314 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 4315 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 4316 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 4317 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4318 4319 /* We do not have a way to disarm Queue causes while leaving 4320 * interrupt enabled for all other causes, ideally 4321 * interrupt should be disabled while we are in NAPI but 4322 * this is not a performance path and napi_schedule() 4323 * can deal with rescheduling. 4324 */ 4325 if (!test_bit(__I40E_DOWN, pf->state)) 4326 napi_schedule_irqoff(&q_vector->napi); 4327 } 4328 4329 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 4330 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4331 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4332 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4333 } 4334 4335 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4336 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4337 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4338 } 4339 4340 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4341 /* disable any further VFLR event notifications */ 4342 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) { 4343 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4344 4345 reg &= ~I40E_PFINT_ICR0_VFLR_MASK; 4346 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4347 } else { 4348 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4349 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4350 } 4351 } 4352 4353 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4354 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4355 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4356 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4357 val = rd32(hw, I40E_GLGEN_RSTAT); 4358 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val); 4359 if (val == I40E_RESET_CORER) { 4360 pf->corer_count++; 4361 } else if (val == I40E_RESET_GLOBR) { 4362 pf->globr_count++; 4363 } else if (val == I40E_RESET_EMPR) { 4364 pf->empr_count++; 4365 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4366 } 4367 } 4368 4369 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4370 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4371 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4372 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4373 rd32(hw, I40E_PFHMC_ERRORINFO), 4374 rd32(hw, I40E_PFHMC_ERRORDATA)); 4375 } 4376 4377 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4378 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4379 4380 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK) 4381 schedule_work(&pf->ptp_extts0_work); 4382 4383 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) 4384 i40e_ptp_tx_hwtstamp(pf); 4385 4386 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4387 } 4388 4389 /* If a critical error is pending we have no choice but to reset the 4390 * device. 4391 * Report and mask out any remaining unexpected interrupts. 4392 */ 4393 icr0_remaining = icr0 & ena_mask; 4394 if (icr0_remaining) { 4395 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4396 icr0_remaining); 4397 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4398 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4399 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4400 dev_info(&pf->pdev->dev, "device will be reset\n"); 4401 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4402 i40e_service_event_schedule(pf); 4403 } 4404 ena_mask &= ~icr0_remaining; 4405 } 4406 ret = IRQ_HANDLED; 4407 4408enable_intr: 4409 /* re-enable interrupt causes */ 4410 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4411 if (!test_bit(__I40E_DOWN, pf->state) || 4412 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4413 i40e_service_event_schedule(pf); 4414 i40e_irq_dynamic_enable_icr0(pf); 4415 } 4416 4417 return ret; 4418} 4419 4420/** 4421 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4422 * @tx_ring: tx ring to clean 4423 * @budget: how many cleans we're allowed 4424 * 4425 * Returns true if there's any budget left (e.g. the clean is finished) 4426 **/ 4427static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4428{ 4429 struct i40e_vsi *vsi = tx_ring->vsi; 4430 u16 i = tx_ring->next_to_clean; 4431 struct i40e_tx_buffer *tx_buf; 4432 struct i40e_tx_desc *tx_desc; 4433 4434 tx_buf = &tx_ring->tx_bi[i]; 4435 tx_desc = I40E_TX_DESC(tx_ring, i); 4436 i -= tx_ring->count; 4437 4438 do { 4439 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4440 4441 /* if next_to_watch is not set then there is no work pending */ 4442 if (!eop_desc) 4443 break; 4444 4445 /* prevent any other reads prior to eop_desc */ 4446 smp_rmb(); 4447 4448 /* if the descriptor isn't done, no work yet to do */ 4449 if (!(eop_desc->cmd_type_offset_bsz & 4450 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4451 break; 4452 4453 /* clear next_to_watch to prevent false hangs */ 4454 tx_buf->next_to_watch = NULL; 4455 4456 tx_desc->buffer_addr = 0; 4457 tx_desc->cmd_type_offset_bsz = 0; 4458 /* move past filter desc */ 4459 tx_buf++; 4460 tx_desc++; 4461 i++; 4462 if (unlikely(!i)) { 4463 i -= tx_ring->count; 4464 tx_buf = tx_ring->tx_bi; 4465 tx_desc = I40E_TX_DESC(tx_ring, 0); 4466 } 4467 /* unmap skb header data */ 4468 dma_unmap_single(tx_ring->dev, 4469 dma_unmap_addr(tx_buf, dma), 4470 dma_unmap_len(tx_buf, len), 4471 DMA_TO_DEVICE); 4472 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4473 kfree(tx_buf->raw_buf); 4474 4475 tx_buf->raw_buf = NULL; 4476 tx_buf->tx_flags = 0; 4477 tx_buf->next_to_watch = NULL; 4478 dma_unmap_len_set(tx_buf, len, 0); 4479 tx_desc->buffer_addr = 0; 4480 tx_desc->cmd_type_offset_bsz = 0; 4481 4482 /* move us past the eop_desc for start of next FD desc */ 4483 tx_buf++; 4484 tx_desc++; 4485 i++; 4486 if (unlikely(!i)) { 4487 i -= tx_ring->count; 4488 tx_buf = tx_ring->tx_bi; 4489 tx_desc = I40E_TX_DESC(tx_ring, 0); 4490 } 4491 4492 /* update budget accounting */ 4493 budget--; 4494 } while (likely(budget)); 4495 4496 i += tx_ring->count; 4497 tx_ring->next_to_clean = i; 4498 4499 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) 4500 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4501 4502 return budget > 0; 4503} 4504 4505/** 4506 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4507 * @irq: interrupt number 4508 * @data: pointer to a q_vector 4509 **/ 4510static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4511{ 4512 struct i40e_q_vector *q_vector = data; 4513 struct i40e_vsi *vsi; 4514 4515 if (!q_vector->tx.ring) 4516 return IRQ_HANDLED; 4517 4518 vsi = q_vector->tx.ring->vsi; 4519 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4520 4521 return IRQ_HANDLED; 4522} 4523 4524/** 4525 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4526 * @vsi: the VSI being configured 4527 * @v_idx: vector index 4528 * @qp_idx: queue pair index 4529 **/ 4530static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4531{ 4532 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4533 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4534 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4535 4536 tx_ring->q_vector = q_vector; 4537 tx_ring->next = q_vector->tx.ring; 4538 q_vector->tx.ring = tx_ring; 4539 q_vector->tx.count++; 4540 4541 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4542 if (i40e_enabled_xdp_vsi(vsi)) { 4543 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4544 4545 xdp_ring->q_vector = q_vector; 4546 xdp_ring->next = q_vector->tx.ring; 4547 q_vector->tx.ring = xdp_ring; 4548 q_vector->tx.count++; 4549 } 4550 4551 rx_ring->q_vector = q_vector; 4552 rx_ring->next = q_vector->rx.ring; 4553 q_vector->rx.ring = rx_ring; 4554 q_vector->rx.count++; 4555} 4556 4557/** 4558 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4559 * @vsi: the VSI being configured 4560 * 4561 * This function maps descriptor rings to the queue-specific vectors 4562 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4563 * one vector per queue pair, but on a constrained vector budget, we 4564 * group the queue pairs as "efficiently" as possible. 4565 **/ 4566static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4567{ 4568 int qp_remaining = vsi->num_queue_pairs; 4569 int q_vectors = vsi->num_q_vectors; 4570 int num_ringpairs; 4571 int v_start = 0; 4572 int qp_idx = 0; 4573 4574 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4575 * group them so there are multiple queues per vector. 4576 * It is also important to go through all the vectors available to be 4577 * sure that if we don't use all the vectors, that the remaining vectors 4578 * are cleared. This is especially important when decreasing the 4579 * number of queues in use. 4580 */ 4581 for (; v_start < q_vectors; v_start++) { 4582 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4583 4584 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4585 4586 q_vector->num_ringpairs = num_ringpairs; 4587 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4588 4589 q_vector->rx.count = 0; 4590 q_vector->tx.count = 0; 4591 q_vector->rx.ring = NULL; 4592 q_vector->tx.ring = NULL; 4593 4594 while (num_ringpairs--) { 4595 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4596 qp_idx++; 4597 qp_remaining--; 4598 } 4599 } 4600} 4601 4602/** 4603 * i40e_vsi_request_irq - Request IRQ from the OS 4604 * @vsi: the VSI being configured 4605 * @basename: name for the vector 4606 **/ 4607static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4608{ 4609 struct i40e_pf *pf = vsi->back; 4610 int err; 4611 4612 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 4613 err = i40e_vsi_request_irq_msix(vsi, basename); 4614 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) 4615 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4616 pf->int_name, pf); 4617 else 4618 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4619 pf->int_name, pf); 4620 4621 if (err) 4622 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4623 4624 return err; 4625} 4626 4627#ifdef CONFIG_NET_POLL_CONTROLLER 4628/** 4629 * i40e_netpoll - A Polling 'interrupt' handler 4630 * @netdev: network interface device structure 4631 * 4632 * This is used by netconsole to send skbs without having to re-enable 4633 * interrupts. It's not called while the normal interrupt routine is executing. 4634 **/ 4635static void i40e_netpoll(struct net_device *netdev) 4636{ 4637 struct i40e_netdev_priv *np = netdev_priv(netdev); 4638 struct i40e_vsi *vsi = np->vsi; 4639 struct i40e_pf *pf = vsi->back; 4640 int i; 4641 4642 /* if interface is down do nothing */ 4643 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4644 return; 4645 4646 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4647 for (i = 0; i < vsi->num_q_vectors; i++) 4648 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4649 } else { 4650 i40e_intr(pf->pdev->irq, netdev); 4651 } 4652} 4653#endif 4654 4655#define I40E_QTX_ENA_WAIT_COUNT 50 4656 4657/** 4658 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4659 * @pf: the PF being configured 4660 * @pf_q: the PF queue 4661 * @enable: enable or disable state of the queue 4662 * 4663 * This routine will wait for the given Tx queue of the PF to reach the 4664 * enabled or disabled state. 4665 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4666 * multiple retries; else will return 0 in case of success. 4667 **/ 4668static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4669{ 4670 int i; 4671 u32 tx_reg; 4672 4673 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4674 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4675 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4676 break; 4677 4678 usleep_range(10, 20); 4679 } 4680 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4681 return -ETIMEDOUT; 4682 4683 return 0; 4684} 4685 4686/** 4687 * i40e_control_tx_q - Start or stop a particular Tx queue 4688 * @pf: the PF structure 4689 * @pf_q: the PF queue to configure 4690 * @enable: start or stop the queue 4691 * 4692 * This function enables or disables a single queue. Note that any delay 4693 * required after the operation is expected to be handled by the caller of 4694 * this function. 4695 **/ 4696static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4697{ 4698 struct i40e_hw *hw = &pf->hw; 4699 u32 tx_reg; 4700 int i; 4701 4702 /* warn the TX unit of coming changes */ 4703 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4704 if (!enable) 4705 usleep_range(10, 20); 4706 4707 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4708 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4709 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4710 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4711 break; 4712 usleep_range(1000, 2000); 4713 } 4714 4715 /* Skip if the queue is already in the requested state */ 4716 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4717 return; 4718 4719 /* turn on/off the queue */ 4720 if (enable) { 4721 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4722 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4723 } else { 4724 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4725 } 4726 4727 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4728} 4729 4730/** 4731 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4732 * @seid: VSI SEID 4733 * @pf: the PF structure 4734 * @pf_q: the PF queue to configure 4735 * @is_xdp: true if the queue is used for XDP 4736 * @enable: start or stop the queue 4737 **/ 4738int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4739 bool is_xdp, bool enable) 4740{ 4741 int ret; 4742 4743 i40e_control_tx_q(pf, pf_q, enable); 4744 4745 /* wait for the change to finish */ 4746 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4747 if (ret) { 4748 dev_info(&pf->pdev->dev, 4749 "VSI seid %d %sTx ring %d %sable timeout\n", 4750 seid, (is_xdp ? "XDP " : ""), pf_q, 4751 (enable ? "en" : "dis")); 4752 } 4753 4754 return ret; 4755} 4756 4757/** 4758 * i40e_vsi_enable_tx - Start a VSI's rings 4759 * @vsi: the VSI being configured 4760 **/ 4761static int i40e_vsi_enable_tx(struct i40e_vsi *vsi) 4762{ 4763 struct i40e_pf *pf = vsi->back; 4764 int i, pf_q, ret = 0; 4765 4766 pf_q = vsi->base_queue; 4767 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4768 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4769 pf_q, 4770 false /*is xdp*/, true); 4771 if (ret) 4772 break; 4773 4774 if (!i40e_enabled_xdp_vsi(vsi)) 4775 continue; 4776 4777 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4778 pf_q + vsi->alloc_queue_pairs, 4779 true /*is xdp*/, true); 4780 if (ret) 4781 break; 4782 } 4783 return ret; 4784} 4785 4786/** 4787 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4788 * @pf: the PF being configured 4789 * @pf_q: the PF queue 4790 * @enable: enable or disable state of the queue 4791 * 4792 * This routine will wait for the given Rx queue of the PF to reach the 4793 * enabled or disabled state. 4794 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4795 * multiple retries; else will return 0 in case of success. 4796 **/ 4797static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4798{ 4799 int i; 4800 u32 rx_reg; 4801 4802 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4803 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4804 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4805 break; 4806 4807 usleep_range(10, 20); 4808 } 4809 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4810 return -ETIMEDOUT; 4811 4812 return 0; 4813} 4814 4815/** 4816 * i40e_control_rx_q - Start or stop a particular Rx queue 4817 * @pf: the PF structure 4818 * @pf_q: the PF queue to configure 4819 * @enable: start or stop the queue 4820 * 4821 * This function enables or disables a single queue. Note that 4822 * any delay required after the operation is expected to be 4823 * handled by the caller of this function. 4824 **/ 4825static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4826{ 4827 struct i40e_hw *hw = &pf->hw; 4828 u32 rx_reg; 4829 int i; 4830 4831 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4832 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4833 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4834 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4835 break; 4836 usleep_range(1000, 2000); 4837 } 4838 4839 /* Skip if the queue is already in the requested state */ 4840 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4841 return; 4842 4843 /* turn on/off the queue */ 4844 if (enable) 4845 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4846 else 4847 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4848 4849 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4850} 4851 4852/** 4853 * i40e_control_wait_rx_q 4854 * @pf: the PF structure 4855 * @pf_q: queue being configured 4856 * @enable: start or stop the rings 4857 * 4858 * This function enables or disables a single queue along with waiting 4859 * for the change to finish. The caller of this function should handle 4860 * the delays needed in the case of disabling queues. 4861 **/ 4862int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4863{ 4864 int ret = 0; 4865 4866 i40e_control_rx_q(pf, pf_q, enable); 4867 4868 /* wait for the change to finish */ 4869 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4870 if (ret) 4871 return ret; 4872 4873 return ret; 4874} 4875 4876/** 4877 * i40e_vsi_enable_rx - Start a VSI's rings 4878 * @vsi: the VSI being configured 4879 **/ 4880static int i40e_vsi_enable_rx(struct i40e_vsi *vsi) 4881{ 4882 struct i40e_pf *pf = vsi->back; 4883 int i, pf_q, ret = 0; 4884 4885 pf_q = vsi->base_queue; 4886 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4887 ret = i40e_control_wait_rx_q(pf, pf_q, true); 4888 if (ret) { 4889 dev_info(&pf->pdev->dev, 4890 "VSI seid %d Rx ring %d enable timeout\n", 4891 vsi->seid, pf_q); 4892 break; 4893 } 4894 } 4895 4896 return ret; 4897} 4898 4899/** 4900 * i40e_vsi_start_rings - Start a VSI's rings 4901 * @vsi: the VSI being configured 4902 **/ 4903int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4904{ 4905 int ret = 0; 4906 4907 /* do rx first for enable and last for disable */ 4908 ret = i40e_vsi_enable_rx(vsi); 4909 if (ret) 4910 return ret; 4911 ret = i40e_vsi_enable_tx(vsi); 4912 4913 return ret; 4914} 4915 4916#define I40E_DISABLE_TX_GAP_MSEC 50 4917 4918/** 4919 * i40e_vsi_stop_rings - Stop a VSI's rings 4920 * @vsi: the VSI being configured 4921 **/ 4922void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4923{ 4924 struct i40e_pf *pf = vsi->back; 4925 u32 pf_q, tx_q_end, rx_q_end; 4926 4927 /* When port TX is suspended, don't wait */ 4928 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4929 return i40e_vsi_stop_rings_no_wait(vsi); 4930 4931 tx_q_end = vsi->base_queue + 4932 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 4933 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4934 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false); 4935 4936 rx_q_end = vsi->base_queue + vsi->num_queue_pairs; 4937 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++) 4938 i40e_control_rx_q(pf, pf_q, false); 4939 4940 msleep(I40E_DISABLE_TX_GAP_MSEC); 4941 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++) 4942 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0); 4943 4944 i40e_vsi_wait_queues_disabled(vsi); 4945} 4946 4947/** 4948 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4949 * @vsi: the VSI being shutdown 4950 * 4951 * This function stops all the rings for a VSI but does not delay to verify 4952 * that rings have been disabled. It is expected that the caller is shutting 4953 * down multiple VSIs at once and will delay together for all the VSIs after 4954 * initiating the shutdown. This is particularly useful for shutting down lots 4955 * of VFs together. Otherwise, a large delay can be incurred while configuring 4956 * each VSI in serial. 4957 **/ 4958void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4959{ 4960 struct i40e_pf *pf = vsi->back; 4961 int i, pf_q; 4962 4963 pf_q = vsi->base_queue; 4964 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4965 i40e_control_tx_q(pf, pf_q, false); 4966 i40e_control_rx_q(pf, pf_q, false); 4967 } 4968} 4969 4970/** 4971 * i40e_vsi_free_irq - Free the irq association with the OS 4972 * @vsi: the VSI being configured 4973 **/ 4974static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4975{ 4976 struct i40e_pf *pf = vsi->back; 4977 struct i40e_hw *hw = &pf->hw; 4978 int base = vsi->base_vector; 4979 u32 val, qp; 4980 int i; 4981 4982 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 4983 if (!vsi->q_vectors) 4984 return; 4985 4986 if (!vsi->irqs_ready) 4987 return; 4988 4989 vsi->irqs_ready = false; 4990 for (i = 0; i < vsi->num_q_vectors; i++) { 4991 int irq_num; 4992 u16 vector; 4993 4994 vector = i + base; 4995 irq_num = pf->msix_entries[vector].vector; 4996 4997 /* free only the irqs that were actually requested */ 4998 if (!vsi->q_vectors[i] || 4999 !vsi->q_vectors[i]->num_ringpairs) 5000 continue; 5001 5002 /* clear the affinity notifier in the IRQ descriptor */ 5003 irq_set_affinity_notifier(irq_num, NULL); 5004 /* remove our suggested affinity mask for this IRQ */ 5005 irq_update_affinity_hint(irq_num, NULL); 5006 free_irq(irq_num, vsi->q_vectors[i]); 5007 5008 /* Tear down the interrupt queue link list 5009 * 5010 * We know that they come in pairs and always 5011 * the Rx first, then the Tx. To clear the 5012 * link list, stick the EOL value into the 5013 * next_q field of the registers. 5014 */ 5015 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 5016 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, 5017 val); 5018 val |= I40E_QUEUE_END_OF_LIST 5019 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 5020 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 5021 5022 while (qp != I40E_QUEUE_END_OF_LIST) { 5023 u32 next; 5024 5025 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5026 5027 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5028 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5029 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5030 I40E_QINT_RQCTL_INTEVENT_MASK); 5031 5032 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5033 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5034 5035 wr32(hw, I40E_QINT_RQCTL(qp), val); 5036 5037 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5038 5039 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK, 5040 val); 5041 5042 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5043 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5044 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5045 I40E_QINT_TQCTL_INTEVENT_MASK); 5046 5047 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5048 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5049 5050 wr32(hw, I40E_QINT_TQCTL(qp), val); 5051 qp = next; 5052 } 5053 } 5054 } else { 5055 free_irq(pf->pdev->irq, pf); 5056 5057 val = rd32(hw, I40E_PFINT_LNKLST0); 5058 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val); 5059 val |= I40E_QUEUE_END_OF_LIST 5060 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5061 wr32(hw, I40E_PFINT_LNKLST0, val); 5062 5063 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5064 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5065 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5066 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5067 I40E_QINT_RQCTL_INTEVENT_MASK); 5068 5069 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5070 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5071 5072 wr32(hw, I40E_QINT_RQCTL(qp), val); 5073 5074 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5075 5076 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5077 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5078 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5079 I40E_QINT_TQCTL_INTEVENT_MASK); 5080 5081 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5082 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5083 5084 wr32(hw, I40E_QINT_TQCTL(qp), val); 5085 } 5086} 5087 5088/** 5089 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 5090 * @vsi: the VSI being configured 5091 * @v_idx: Index of vector to be freed 5092 * 5093 * This function frees the memory allocated to the q_vector. In addition if 5094 * NAPI is enabled it will delete any references to the NAPI struct prior 5095 * to freeing the q_vector. 5096 **/ 5097static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 5098{ 5099 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 5100 struct i40e_ring *ring; 5101 5102 if (!q_vector) 5103 return; 5104 5105 /* disassociate q_vector from rings */ 5106 i40e_for_each_ring(ring, q_vector->tx) 5107 ring->q_vector = NULL; 5108 5109 i40e_for_each_ring(ring, q_vector->rx) 5110 ring->q_vector = NULL; 5111 5112 /* only VSI w/ an associated netdev is set up w/ NAPI */ 5113 if (vsi->netdev) 5114 netif_napi_del(&q_vector->napi); 5115 5116 vsi->q_vectors[v_idx] = NULL; 5117 5118 kfree_rcu(q_vector, rcu); 5119} 5120 5121/** 5122 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 5123 * @vsi: the VSI being un-configured 5124 * 5125 * This frees the memory allocated to the q_vectors and 5126 * deletes references to the NAPI struct. 5127 **/ 5128static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 5129{ 5130 int v_idx; 5131 5132 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 5133 i40e_free_q_vector(vsi, v_idx); 5134} 5135 5136/** 5137 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 5138 * @pf: board private structure 5139 **/ 5140static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 5141{ 5142 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 5143 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 5144 pci_disable_msix(pf->pdev); 5145 kfree(pf->msix_entries); 5146 pf->msix_entries = NULL; 5147 kfree(pf->irq_pile); 5148 pf->irq_pile = NULL; 5149 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 5150 pci_disable_msi(pf->pdev); 5151 } 5152 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 5153 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 5154} 5155 5156/** 5157 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 5158 * @pf: board private structure 5159 * 5160 * We go through and clear interrupt specific resources and reset the structure 5161 * to pre-load conditions 5162 **/ 5163static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 5164{ 5165 struct i40e_vsi *vsi; 5166 int i; 5167 5168 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) 5169 i40e_free_misc_vector(pf); 5170 5171 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 5172 I40E_IWARP_IRQ_PILE_ID); 5173 5174 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 5175 5176 i40e_pf_for_each_vsi(pf, i, vsi) 5177 i40e_vsi_free_q_vectors(vsi); 5178 5179 i40e_reset_interrupt_capability(pf); 5180} 5181 5182/** 5183 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 5184 * @vsi: the VSI being configured 5185 **/ 5186static void i40e_napi_enable_all(struct i40e_vsi *vsi) 5187{ 5188 int q_idx; 5189 5190 if (!vsi->netdev) 5191 return; 5192 5193 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5194 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5195 5196 if (q_vector->rx.ring || q_vector->tx.ring) 5197 napi_enable(&q_vector->napi); 5198 } 5199} 5200 5201/** 5202 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5203 * @vsi: the VSI being configured 5204 **/ 5205static void i40e_napi_disable_all(struct i40e_vsi *vsi) 5206{ 5207 int q_idx; 5208 5209 if (!vsi->netdev) 5210 return; 5211 5212 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5213 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5214 5215 if (q_vector->rx.ring || q_vector->tx.ring) 5216 napi_disable(&q_vector->napi); 5217 } 5218} 5219 5220/** 5221 * i40e_vsi_close - Shut down a VSI 5222 * @vsi: the vsi to be quelled 5223 **/ 5224static void i40e_vsi_close(struct i40e_vsi *vsi) 5225{ 5226 struct i40e_pf *pf = vsi->back; 5227 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 5228 i40e_down(vsi); 5229 i40e_vsi_free_irq(vsi); 5230 i40e_vsi_free_tx_resources(vsi); 5231 i40e_vsi_free_rx_resources(vsi); 5232 vsi->current_netdev_flags = 0; 5233 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 5234 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 5235 set_bit(__I40E_CLIENT_RESET, pf->state); 5236} 5237 5238/** 5239 * i40e_quiesce_vsi - Pause a given VSI 5240 * @vsi: the VSI being paused 5241 **/ 5242static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 5243{ 5244 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 5245 return; 5246 5247 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 5248 if (vsi->netdev && netif_running(vsi->netdev)) 5249 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 5250 else 5251 i40e_vsi_close(vsi); 5252} 5253 5254/** 5255 * i40e_unquiesce_vsi - Resume a given VSI 5256 * @vsi: the VSI being resumed 5257 **/ 5258static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 5259{ 5260 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 5261 return; 5262 5263 if (vsi->netdev && netif_running(vsi->netdev)) 5264 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 5265 else 5266 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 5267} 5268 5269/** 5270 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 5271 * @pf: the PF 5272 **/ 5273static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 5274{ 5275 struct i40e_vsi *vsi; 5276 int v; 5277 5278 i40e_pf_for_each_vsi(pf, v, vsi) 5279 i40e_quiesce_vsi(vsi); 5280} 5281 5282/** 5283 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 5284 * @pf: the PF 5285 **/ 5286static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 5287{ 5288 struct i40e_vsi *vsi; 5289 int v; 5290 5291 i40e_pf_for_each_vsi(pf, v, vsi) 5292 i40e_unquiesce_vsi(vsi); 5293} 5294 5295/** 5296 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 5297 * @vsi: the VSI being configured 5298 * 5299 * Wait until all queues on a given VSI have been disabled. 5300 **/ 5301int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 5302{ 5303 struct i40e_pf *pf = vsi->back; 5304 int i, pf_q, ret; 5305 5306 pf_q = vsi->base_queue; 5307 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 5308 /* Check and wait for the Tx queue */ 5309 ret = i40e_pf_txq_wait(pf, pf_q, false); 5310 if (ret) { 5311 dev_info(&pf->pdev->dev, 5312 "VSI seid %d Tx ring %d disable timeout\n", 5313 vsi->seid, pf_q); 5314 return ret; 5315 } 5316 5317 if (!i40e_enabled_xdp_vsi(vsi)) 5318 goto wait_rx; 5319 5320 /* Check and wait for the XDP Tx queue */ 5321 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 5322 false); 5323 if (ret) { 5324 dev_info(&pf->pdev->dev, 5325 "VSI seid %d XDP Tx ring %d disable timeout\n", 5326 vsi->seid, pf_q); 5327 return ret; 5328 } 5329wait_rx: 5330 /* Check and wait for the Rx queue */ 5331 ret = i40e_pf_rxq_wait(pf, pf_q, false); 5332 if (ret) { 5333 dev_info(&pf->pdev->dev, 5334 "VSI seid %d Rx ring %d disable timeout\n", 5335 vsi->seid, pf_q); 5336 return ret; 5337 } 5338 } 5339 5340 return 0; 5341} 5342 5343#ifdef CONFIG_I40E_DCB 5344/** 5345 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 5346 * @pf: the PF 5347 * 5348 * This function waits for the queues to be in disabled state for all the 5349 * VSIs that are managed by this PF. 5350 **/ 5351static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5352{ 5353 struct i40e_vsi *vsi; 5354 int v, ret = 0; 5355 5356 i40e_pf_for_each_vsi(pf, v, vsi) { 5357 ret = i40e_vsi_wait_queues_disabled(vsi); 5358 if (ret) 5359 break; 5360 } 5361 5362 return ret; 5363} 5364 5365#endif 5366 5367/** 5368 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5369 * @pf: pointer to PF 5370 * 5371 * Get TC map for ISCSI PF type that will include iSCSI TC 5372 * and LAN TC. 5373 **/ 5374static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5375{ 5376 struct i40e_dcb_app_priority_table app; 5377 struct i40e_hw *hw = &pf->hw; 5378 u8 enabled_tc = 1; /* TC0 is always enabled */ 5379 u8 tc, i; 5380 /* Get the iSCSI APP TLV */ 5381 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5382 5383 for (i = 0; i < dcbcfg->numapps; i++) { 5384 app = dcbcfg->app[i]; 5385 if (app.selector == I40E_APP_SEL_TCPIP && 5386 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5387 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5388 enabled_tc |= BIT(tc); 5389 break; 5390 } 5391 } 5392 5393 return enabled_tc; 5394} 5395 5396/** 5397 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5398 * @dcbcfg: the corresponding DCBx configuration structure 5399 * 5400 * Return the number of TCs from given DCBx configuration 5401 **/ 5402static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5403{ 5404 int i, tc_unused = 0; 5405 u8 num_tc = 0; 5406 u8 ret = 0; 5407 5408 /* Scan the ETS Config Priority Table to find 5409 * traffic class enabled for a given priority 5410 * and create a bitmask of enabled TCs 5411 */ 5412 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5413 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5414 5415 /* Now scan the bitmask to check for 5416 * contiguous TCs starting with TC0 5417 */ 5418 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5419 if (num_tc & BIT(i)) { 5420 if (!tc_unused) { 5421 ret++; 5422 } else { 5423 pr_err("Non-contiguous TC - Disabling DCB\n"); 5424 return 1; 5425 } 5426 } else { 5427 tc_unused = 1; 5428 } 5429 } 5430 5431 /* There is always at least TC0 */ 5432 if (!ret) 5433 ret = 1; 5434 5435 return ret; 5436} 5437 5438/** 5439 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5440 * @dcbcfg: the corresponding DCBx configuration structure 5441 * 5442 * Query the current DCB configuration and return the number of 5443 * traffic classes enabled from the given DCBX config 5444 **/ 5445static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5446{ 5447 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5448 u8 enabled_tc = 1; 5449 u8 i; 5450 5451 for (i = 0; i < num_tc; i++) 5452 enabled_tc |= BIT(i); 5453 5454 return enabled_tc; 5455} 5456 5457/** 5458 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5459 * @pf: PF being queried 5460 * 5461 * Query the current MQPRIO configuration and return the number of 5462 * traffic classes enabled. 5463 **/ 5464static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5465{ 5466 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5467 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5468 u8 enabled_tc = 1, i; 5469 5470 for (i = 1; i < num_tc; i++) 5471 enabled_tc |= BIT(i); 5472 return enabled_tc; 5473} 5474 5475/** 5476 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5477 * @pf: PF being queried 5478 * 5479 * Return number of traffic classes enabled for the given PF 5480 **/ 5481static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5482{ 5483 u8 i, enabled_tc = 1; 5484 u8 num_tc = 0; 5485 5486 if (i40e_is_tc_mqprio_enabled(pf)) { 5487 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 5488 5489 return vsi->mqprio_qopt.qopt.num_tc; 5490 } 5491 5492 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5493 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5494 return 1; 5495 5496 /* SFP mode will be enabled for all TCs on port */ 5497 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5498 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config); 5499 5500 /* MFP mode return count of enabled TCs for this PF */ 5501 if (pf->hw.func_caps.iscsi) 5502 enabled_tc = i40e_get_iscsi_tc_map(pf); 5503 else 5504 return 1; /* Only TC0 */ 5505 5506 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5507 if (enabled_tc & BIT(i)) 5508 num_tc++; 5509 } 5510 return num_tc; 5511} 5512 5513/** 5514 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes 5515 * @pf: PF being queried 5516 * 5517 * Return a bitmap for enabled traffic classes for this PF. 5518 **/ 5519static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5520{ 5521 if (i40e_is_tc_mqprio_enabled(pf)) 5522 return i40e_mqprio_get_enabled_tc(pf); 5523 5524 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5525 * default TC 5526 */ 5527 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) 5528 return I40E_DEFAULT_TRAFFIC_CLASS; 5529 5530 /* SFP mode we want PF to be enabled for all TCs */ 5531 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 5532 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5533 5534 /* MFP enabled and iSCSI PF type */ 5535 if (pf->hw.func_caps.iscsi) 5536 return i40e_get_iscsi_tc_map(pf); 5537 else 5538 return I40E_DEFAULT_TRAFFIC_CLASS; 5539} 5540 5541/** 5542 * i40e_vsi_get_bw_info - Query VSI BW Information 5543 * @vsi: the VSI being queried 5544 * 5545 * Returns 0 on success, negative value on failure 5546 **/ 5547static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5548{ 5549 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5550 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5551 struct i40e_pf *pf = vsi->back; 5552 struct i40e_hw *hw = &pf->hw; 5553 u32 tc_bw_max; 5554 int ret; 5555 int i; 5556 5557 /* Get the VSI level BW configuration */ 5558 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5559 if (ret) { 5560 dev_info(&pf->pdev->dev, 5561 "couldn't get PF vsi bw config, err %pe aq_err %s\n", 5562 ERR_PTR(ret), 5563 libie_aq_str(pf->hw.aq.asq_last_status)); 5564 return -EINVAL; 5565 } 5566 5567 /* Get the VSI level BW configuration per TC */ 5568 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5569 NULL); 5570 if (ret) { 5571 dev_info(&pf->pdev->dev, 5572 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n", 5573 ERR_PTR(ret), 5574 libie_aq_str(pf->hw.aq.asq_last_status)); 5575 return -EINVAL; 5576 } 5577 5578 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5579 dev_info(&pf->pdev->dev, 5580 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5581 bw_config.tc_valid_bits, 5582 bw_ets_config.tc_valid_bits); 5583 /* Still continuing */ 5584 } 5585 5586 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5587 vsi->bw_max_quanta = bw_config.max_bw; 5588 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5589 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5590 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5591 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5592 vsi->bw_ets_limit_credits[i] = 5593 le16_to_cpu(bw_ets_config.credits[i]); 5594 /* 3 bits out of 4 for each TC */ 5595 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5596 } 5597 5598 return 0; 5599} 5600 5601/** 5602 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5603 * @vsi: the VSI being configured 5604 * @enabled_tc: TC bitmap 5605 * @bw_share: BW shared credits per TC 5606 * 5607 * Returns 0 on success, negative value on failure 5608 **/ 5609static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5610 u8 *bw_share) 5611{ 5612 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5613 struct i40e_pf *pf = vsi->back; 5614 int ret; 5615 int i; 5616 5617 /* There is no need to reset BW when mqprio mode is on. */ 5618 if (i40e_is_tc_mqprio_enabled(pf)) 5619 return 0; 5620 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 5621 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5622 if (ret) 5623 dev_info(&pf->pdev->dev, 5624 "Failed to reset tx rate for vsi->seid %u\n", 5625 vsi->seid); 5626 return ret; 5627 } 5628 memset(&bw_data, 0, sizeof(bw_data)); 5629 bw_data.tc_valid_bits = enabled_tc; 5630 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5631 bw_data.tc_bw_credits[i] = bw_share[i]; 5632 5633 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5634 if (ret) { 5635 dev_info(&pf->pdev->dev, 5636 "AQ command Config VSI BW allocation per TC failed = %d\n", 5637 pf->hw.aq.asq_last_status); 5638 return -EINVAL; 5639 } 5640 5641 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5642 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5643 5644 return 0; 5645} 5646 5647/** 5648 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5649 * @vsi: the VSI being configured 5650 * @enabled_tc: TC map to be enabled 5651 * 5652 **/ 5653static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5654{ 5655 struct net_device *netdev = vsi->netdev; 5656 struct i40e_pf *pf = vsi->back; 5657 struct i40e_hw *hw = &pf->hw; 5658 u8 netdev_tc = 0; 5659 int i; 5660 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5661 5662 if (!netdev) 5663 return; 5664 5665 if (!enabled_tc) { 5666 netdev_reset_tc(netdev); 5667 return; 5668 } 5669 5670 /* Set up actual enabled TCs on the VSI */ 5671 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5672 return; 5673 5674 /* set per TC queues for the VSI */ 5675 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5676 /* Only set TC queues for enabled tcs 5677 * 5678 * e.g. For a VSI that has TC0 and TC3 enabled the 5679 * enabled_tc bitmap would be 0x00001001; the driver 5680 * will set the numtc for netdev as 2 that will be 5681 * referenced by the netdev layer as TC 0 and 1. 5682 */ 5683 if (vsi->tc_config.enabled_tc & BIT(i)) 5684 netdev_set_tc_queue(netdev, 5685 vsi->tc_config.tc_info[i].netdev_tc, 5686 vsi->tc_config.tc_info[i].qcount, 5687 vsi->tc_config.tc_info[i].qoffset); 5688 } 5689 5690 if (i40e_is_tc_mqprio_enabled(pf)) 5691 return; 5692 5693 /* Assign UP2TC map for the VSI */ 5694 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5695 /* Get the actual TC# for the UP */ 5696 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5697 /* Get the mapped netdev TC# for the UP */ 5698 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5699 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5700 } 5701} 5702 5703/** 5704 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5705 * @vsi: the VSI being configured 5706 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5707 **/ 5708static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5709 struct i40e_vsi_context *ctxt) 5710{ 5711 /* copy just the sections touched not the entire info 5712 * since not all sections are valid as returned by 5713 * update vsi params 5714 */ 5715 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5716 memcpy(&vsi->info.queue_mapping, 5717 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5718 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5719 sizeof(vsi->info.tc_mapping)); 5720} 5721 5722/** 5723 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI 5724 * @vsi: the VSI being reconfigured 5725 * @vsi_offset: offset from main VF VSI 5726 */ 5727int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset) 5728{ 5729 struct i40e_vsi_context ctxt = {}; 5730 struct i40e_pf *pf; 5731 struct i40e_hw *hw; 5732 int ret; 5733 5734 if (!vsi) 5735 return -EINVAL; 5736 pf = vsi->back; 5737 hw = &pf->hw; 5738 5739 ctxt.seid = vsi->seid; 5740 ctxt.pf_num = hw->pf_id; 5741 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset; 5742 ctxt.uplink_seid = vsi->uplink_seid; 5743 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5744 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 5745 ctxt.info = vsi->info; 5746 5747 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc, 5748 false); 5749 if (vsi->reconfig_rss) { 5750 vsi->rss_size = min_t(int, pf->alloc_rss_size, 5751 vsi->num_queue_pairs); 5752 ret = i40e_vsi_config_rss(vsi); 5753 if (ret) { 5754 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n"); 5755 return ret; 5756 } 5757 vsi->reconfig_rss = false; 5758 } 5759 5760 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5761 if (ret) { 5762 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n", 5763 ERR_PTR(ret), 5764 libie_aq_str(hw->aq.asq_last_status)); 5765 return ret; 5766 } 5767 /* update the local VSI info with updated queue map */ 5768 i40e_vsi_update_queue_map(vsi, &ctxt); 5769 vsi->info.valid_sections = 0; 5770 5771 return ret; 5772} 5773 5774/** 5775 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5776 * @vsi: VSI to be configured 5777 * @enabled_tc: TC bitmap 5778 * 5779 * This configures a particular VSI for TCs that are mapped to the 5780 * given TC bitmap. It uses default bandwidth share for TCs across 5781 * VSIs to configure TC for a particular VSI. 5782 * 5783 * NOTE: 5784 * It is expected that the VSI queues have been quisced before calling 5785 * this function. 5786 **/ 5787static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5788{ 5789 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5790 struct i40e_pf *pf = vsi->back; 5791 struct i40e_hw *hw = &pf->hw; 5792 struct i40e_vsi_context ctxt; 5793 int ret = 0; 5794 int i; 5795 5796 /* Check if enabled_tc is same as existing or new TCs */ 5797 if (vsi->tc_config.enabled_tc == enabled_tc && 5798 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5799 return ret; 5800 5801 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5802 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5803 if (enabled_tc & BIT(i)) 5804 bw_share[i] = 1; 5805 } 5806 5807 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5808 if (ret) { 5809 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5810 5811 dev_info(&pf->pdev->dev, 5812 "Failed configuring TC map %d for VSI %d\n", 5813 enabled_tc, vsi->seid); 5814 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5815 &bw_config, NULL); 5816 if (ret) { 5817 dev_info(&pf->pdev->dev, 5818 "Failed querying vsi bw info, err %pe aq_err %s\n", 5819 ERR_PTR(ret), 5820 libie_aq_str(hw->aq.asq_last_status)); 5821 goto out; 5822 } 5823 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5824 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5825 5826 if (!valid_tc) 5827 valid_tc = bw_config.tc_valid_bits; 5828 /* Always enable TC0, no matter what */ 5829 valid_tc |= 1; 5830 dev_info(&pf->pdev->dev, 5831 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5832 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5833 enabled_tc = valid_tc; 5834 } 5835 5836 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5837 if (ret) { 5838 dev_err(&pf->pdev->dev, 5839 "Unable to configure TC map %d for VSI %d\n", 5840 enabled_tc, vsi->seid); 5841 goto out; 5842 } 5843 } 5844 5845 /* Update Queue Pairs Mapping for currently enabled UPs */ 5846 ctxt.seid = vsi->seid; 5847 ctxt.pf_num = vsi->back->hw.pf_id; 5848 ctxt.vf_num = 0; 5849 ctxt.uplink_seid = vsi->uplink_seid; 5850 ctxt.info = vsi->info; 5851 if (i40e_is_tc_mqprio_enabled(pf)) { 5852 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5853 if (ret) 5854 goto out; 5855 } else { 5856 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5857 } 5858 5859 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5860 * queues changed. 5861 */ 5862 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5863 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5864 vsi->num_queue_pairs); 5865 ret = i40e_vsi_config_rss(vsi); 5866 if (ret) { 5867 dev_info(&vsi->back->pdev->dev, 5868 "Failed to reconfig rss for num_queues\n"); 5869 return ret; 5870 } 5871 vsi->reconfig_rss = false; 5872 } 5873 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 5874 ctxt.info.valid_sections |= 5875 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5876 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5877 } 5878 5879 /* Update the VSI after updating the VSI queue-mapping 5880 * information 5881 */ 5882 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5883 if (ret) { 5884 dev_info(&pf->pdev->dev, 5885 "Update vsi tc config failed, err %pe aq_err %s\n", 5886 ERR_PTR(ret), 5887 libie_aq_str(hw->aq.asq_last_status)); 5888 goto out; 5889 } 5890 /* update the local VSI info with updated queue map */ 5891 i40e_vsi_update_queue_map(vsi, &ctxt); 5892 vsi->info.valid_sections = 0; 5893 5894 /* Update current VSI BW information */ 5895 ret = i40e_vsi_get_bw_info(vsi); 5896 if (ret) { 5897 dev_info(&pf->pdev->dev, 5898 "Failed updating vsi bw info, err %pe aq_err %s\n", 5899 ERR_PTR(ret), 5900 libie_aq_str(hw->aq.asq_last_status)); 5901 goto out; 5902 } 5903 5904 /* Update the netdev TC setup */ 5905 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5906out: 5907 return ret; 5908} 5909 5910/** 5911 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map 5912 * @vsi: VSI to be reconfigured 5913 * 5914 * This reconfigures a particular VSI for TCs that are mapped to the 5915 * TC bitmap stored previously for the VSI. 5916 * 5917 * Context: It is expected that the VSI queues have been quisced before 5918 * calling this function. 5919 * 5920 * Return: 0 on success, negative value on failure 5921 **/ 5922static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi) 5923{ 5924 u8 enabled_tc; 5925 5926 enabled_tc = vsi->tc_config.enabled_tc; 5927 vsi->tc_config.enabled_tc = 0; 5928 5929 return i40e_vsi_config_tc(vsi, enabled_tc); 5930} 5931 5932/** 5933 * i40e_get_link_speed - Returns link speed for the interface 5934 * @vsi: VSI to be configured 5935 * 5936 **/ 5937static int i40e_get_link_speed(struct i40e_vsi *vsi) 5938{ 5939 struct i40e_pf *pf = vsi->back; 5940 5941 switch (pf->hw.phy.link_info.link_speed) { 5942 case I40E_LINK_SPEED_40GB: 5943 return 40000; 5944 case I40E_LINK_SPEED_25GB: 5945 return 25000; 5946 case I40E_LINK_SPEED_20GB: 5947 return 20000; 5948 case I40E_LINK_SPEED_10GB: 5949 return 10000; 5950 case I40E_LINK_SPEED_1GB: 5951 return 1000; 5952 default: 5953 return -EINVAL; 5954 } 5955} 5956 5957/** 5958 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits 5959 * @vsi: Pointer to vsi structure 5960 * @max_tx_rate: max TX rate in bytes to be converted into Mbits 5961 * 5962 * Helper function to convert units before send to set BW limit 5963 **/ 5964static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate) 5965{ 5966 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) { 5967 dev_warn(&vsi->back->pdev->dev, 5968 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5969 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 5970 } else { 5971 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 5972 } 5973 5974 return max_tx_rate; 5975} 5976 5977/** 5978 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5979 * @vsi: VSI to be configured 5980 * @seid: seid of the channel/VSI 5981 * @max_tx_rate: max TX rate to be configured as BW limit 5982 * 5983 * Helper function to set BW limit for a given VSI 5984 **/ 5985int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5986{ 5987 struct i40e_pf *pf = vsi->back; 5988 u64 credits = 0; 5989 int speed = 0; 5990 int ret = 0; 5991 5992 speed = i40e_get_link_speed(vsi); 5993 if (max_tx_rate > speed) { 5994 dev_err(&pf->pdev->dev, 5995 "Invalid max tx rate %llu specified for VSI seid %d.", 5996 max_tx_rate, seid); 5997 return -EINVAL; 5998 } 5999 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) { 6000 dev_warn(&pf->pdev->dev, 6001 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 6002 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 6003 } 6004 6005 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 6006 credits = max_tx_rate; 6007 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6008 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 6009 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 6010 if (ret) 6011 dev_err(&pf->pdev->dev, 6012 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n", 6013 max_tx_rate, seid, ERR_PTR(ret), 6014 libie_aq_str(pf->hw.aq.asq_last_status)); 6015 return ret; 6016} 6017 6018/** 6019 * i40e_remove_queue_channels - Remove queue channels for the TCs 6020 * @vsi: VSI to be configured 6021 * 6022 * Remove queue channels for the TCs 6023 **/ 6024static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 6025{ 6026 struct i40e_cloud_filter *cfilter; 6027 enum libie_aq_err last_aq_status; 6028 struct i40e_channel *ch, *ch_tmp; 6029 struct i40e_pf *pf = vsi->back; 6030 struct hlist_node *node; 6031 int ret, i; 6032 6033 /* Reset rss size that was stored when reconfiguring rss for 6034 * channel VSIs with non-power-of-2 queue count. 6035 */ 6036 vsi->current_rss_size = 0; 6037 6038 /* perform cleanup for channels if they exist */ 6039 if (list_empty(&vsi->ch_list)) 6040 return; 6041 6042 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6043 struct i40e_vsi *p_vsi; 6044 6045 list_del(&ch->list); 6046 p_vsi = ch->parent_vsi; 6047 if (!p_vsi || !ch->initialized) { 6048 kfree(ch); 6049 continue; 6050 } 6051 /* Reset queue contexts */ 6052 for (i = 0; i < ch->num_queue_pairs; i++) { 6053 struct i40e_ring *tx_ring, *rx_ring; 6054 u16 pf_q; 6055 6056 pf_q = ch->base_queue + i; 6057 tx_ring = vsi->tx_rings[pf_q]; 6058 tx_ring->ch = NULL; 6059 6060 rx_ring = vsi->rx_rings[pf_q]; 6061 rx_ring->ch = NULL; 6062 } 6063 6064 /* Reset BW configured for this VSI via mqprio */ 6065 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 6066 if (ret) 6067 dev_info(&vsi->back->pdev->dev, 6068 "Failed to reset tx rate for ch->seid %u\n", 6069 ch->seid); 6070 6071 /* delete cloud filters associated with this channel */ 6072 hlist_for_each_entry_safe(cfilter, node, 6073 &pf->cloud_filter_list, cloud_node) { 6074 if (cfilter->seid != ch->seid) 6075 continue; 6076 6077 hash_del(&cfilter->cloud_node); 6078 if (cfilter->dst_port) 6079 ret = i40e_add_del_cloud_filter_big_buf(vsi, 6080 cfilter, 6081 false); 6082 else 6083 ret = i40e_add_del_cloud_filter(vsi, cfilter, 6084 false); 6085 last_aq_status = pf->hw.aq.asq_last_status; 6086 if (ret) 6087 dev_info(&pf->pdev->dev, 6088 "Failed to delete cloud filter, err %pe aq_err %s\n", 6089 ERR_PTR(ret), 6090 libie_aq_str(last_aq_status)); 6091 kfree(cfilter); 6092 } 6093 6094 /* delete VSI from FW */ 6095 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 6096 NULL); 6097 if (ret) 6098 dev_err(&vsi->back->pdev->dev, 6099 "unable to remove channel (%d) for parent VSI(%d)\n", 6100 ch->seid, p_vsi->seid); 6101 kfree(ch); 6102 } 6103 INIT_LIST_HEAD(&vsi->ch_list); 6104} 6105 6106/** 6107 * i40e_get_max_queues_for_channel 6108 * @vsi: ptr to VSI to which channels are associated with 6109 * 6110 * Helper function which returns max value among the queue counts set on the 6111 * channels/TCs created. 6112 **/ 6113static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 6114{ 6115 struct i40e_channel *ch, *ch_tmp; 6116 int max = 0; 6117 6118 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6119 if (!ch->initialized) 6120 continue; 6121 if (ch->num_queue_pairs > max) 6122 max = ch->num_queue_pairs; 6123 } 6124 6125 return max; 6126} 6127 6128/** 6129 * i40e_validate_num_queues - validate num_queues w.r.t channel 6130 * @pf: ptr to PF device 6131 * @num_queues: number of queues 6132 * @vsi: the parent VSI 6133 * @reconfig_rss: indicates should the RSS be reconfigured or not 6134 * 6135 * This function validates number of queues in the context of new channel 6136 * which is being established and determines if RSS should be reconfigured 6137 * or not for parent VSI. 6138 **/ 6139static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 6140 struct i40e_vsi *vsi, bool *reconfig_rss) 6141{ 6142 int max_ch_queues; 6143 6144 if (!reconfig_rss) 6145 return -EINVAL; 6146 6147 *reconfig_rss = false; 6148 if (vsi->current_rss_size) { 6149 if (num_queues > vsi->current_rss_size) { 6150 dev_dbg(&pf->pdev->dev, 6151 "Error: num_queues (%d) > vsi's current_size(%d)\n", 6152 num_queues, vsi->current_rss_size); 6153 return -EINVAL; 6154 } else if ((num_queues < vsi->current_rss_size) && 6155 (!is_power_of_2(num_queues))) { 6156 dev_dbg(&pf->pdev->dev, 6157 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 6158 num_queues, vsi->current_rss_size); 6159 return -EINVAL; 6160 } 6161 } 6162 6163 if (!is_power_of_2(num_queues)) { 6164 /* Find the max num_queues configured for channel if channel 6165 * exist. 6166 * if channel exist, then enforce 'num_queues' to be more than 6167 * max ever queues configured for channel. 6168 */ 6169 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 6170 if (num_queues < max_ch_queues) { 6171 dev_dbg(&pf->pdev->dev, 6172 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 6173 num_queues, max_ch_queues); 6174 return -EINVAL; 6175 } 6176 *reconfig_rss = true; 6177 } 6178 6179 return 0; 6180} 6181 6182/** 6183 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 6184 * @vsi: the VSI being setup 6185 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 6186 * 6187 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 6188 **/ 6189static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 6190{ 6191 struct i40e_pf *pf = vsi->back; 6192 u8 seed[I40E_HKEY_ARRAY_SIZE]; 6193 struct i40e_hw *hw = &pf->hw; 6194 int local_rss_size; 6195 u8 *lut; 6196 int ret; 6197 6198 if (!vsi->rss_size) 6199 return -EINVAL; 6200 6201 if (rss_size > vsi->rss_size) 6202 return -EINVAL; 6203 6204 local_rss_size = min_t(int, vsi->rss_size, rss_size); 6205 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 6206 if (!lut) 6207 return -ENOMEM; 6208 6209 /* Ignoring user configured lut if there is one */ 6210 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 6211 6212 /* Use user configured hash key if there is one, otherwise 6213 * use default. 6214 */ 6215 if (vsi->rss_hkey_user) 6216 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 6217 else 6218 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 6219 6220 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 6221 if (ret) { 6222 dev_info(&pf->pdev->dev, 6223 "Cannot set RSS lut, err %pe aq_err %s\n", 6224 ERR_PTR(ret), 6225 libie_aq_str(hw->aq.asq_last_status)); 6226 kfree(lut); 6227 return ret; 6228 } 6229 kfree(lut); 6230 6231 /* Do the update w.r.t. storing rss_size */ 6232 if (!vsi->orig_rss_size) 6233 vsi->orig_rss_size = vsi->rss_size; 6234 vsi->current_rss_size = local_rss_size; 6235 6236 return ret; 6237} 6238 6239/** 6240 * i40e_channel_setup_queue_map - Setup a channel queue map 6241 * @pf: ptr to PF device 6242 * @ctxt: VSI context structure 6243 * @ch: ptr to channel structure 6244 * 6245 * Setup queue map for a specific channel 6246 **/ 6247static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 6248 struct i40e_vsi_context *ctxt, 6249 struct i40e_channel *ch) 6250{ 6251 u16 qcount, qmap, sections = 0; 6252 u8 offset = 0; 6253 int pow; 6254 6255 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 6256 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 6257 6258 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 6259 ch->num_queue_pairs = qcount; 6260 6261 /* find the next higher power-of-2 of num queue pairs */ 6262 pow = ilog2(qcount); 6263 if (!is_power_of_2(qcount)) 6264 pow++; 6265 6266 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 6267 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 6268 6269 /* Setup queue TC[0].qmap for given VSI context */ 6270 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 6271 6272 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 6273 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 6274 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 6275 ctxt->info.valid_sections |= cpu_to_le16(sections); 6276} 6277 6278/** 6279 * i40e_add_channel - add a channel by adding VSI 6280 * @pf: ptr to PF device 6281 * @uplink_seid: underlying HW switching element (VEB) ID 6282 * @ch: ptr to channel structure 6283 * 6284 * Add a channel (VSI) using add_vsi and queue_map 6285 **/ 6286static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 6287 struct i40e_channel *ch) 6288{ 6289 struct i40e_hw *hw = &pf->hw; 6290 struct i40e_vsi_context ctxt; 6291 u8 enabled_tc = 0x1; /* TC0 enabled */ 6292 int ret; 6293 6294 if (ch->type != I40E_VSI_VMDQ2) { 6295 dev_info(&pf->pdev->dev, 6296 "add new vsi failed, ch->type %d\n", ch->type); 6297 return -EINVAL; 6298 } 6299 6300 memset(&ctxt, 0, sizeof(ctxt)); 6301 ctxt.pf_num = hw->pf_id; 6302 ctxt.vf_num = 0; 6303 ctxt.uplink_seid = uplink_seid; 6304 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 6305 if (ch->type == I40E_VSI_VMDQ2) 6306 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 6307 6308 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6309 ctxt.info.valid_sections |= 6310 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6311 ctxt.info.switch_id = 6312 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6313 } 6314 6315 /* Set queue map for a given VSI context */ 6316 i40e_channel_setup_queue_map(pf, &ctxt, ch); 6317 6318 /* Now time to create VSI */ 6319 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 6320 if (ret) { 6321 dev_info(&pf->pdev->dev, 6322 "add new vsi failed, err %pe aq_err %s\n", 6323 ERR_PTR(ret), 6324 libie_aq_str(pf->hw.aq.asq_last_status)); 6325 return -ENOENT; 6326 } 6327 6328 /* Success, update channel, set enabled_tc only if the channel 6329 * is not a macvlan 6330 */ 6331 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 6332 ch->seid = ctxt.seid; 6333 ch->vsi_number = ctxt.vsi_number; 6334 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx); 6335 6336 /* copy just the sections touched not the entire info 6337 * since not all sections are valid as returned by 6338 * update vsi params 6339 */ 6340 ch->info.mapping_flags = ctxt.info.mapping_flags; 6341 memcpy(&ch->info.queue_mapping, 6342 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 6343 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 6344 sizeof(ctxt.info.tc_mapping)); 6345 6346 return 0; 6347} 6348 6349static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 6350 u8 *bw_share) 6351{ 6352 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 6353 int ret; 6354 int i; 6355 6356 memset(&bw_data, 0, sizeof(bw_data)); 6357 bw_data.tc_valid_bits = ch->enabled_tc; 6358 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6359 bw_data.tc_bw_credits[i] = bw_share[i]; 6360 6361 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 6362 &bw_data, NULL); 6363 if (ret) { 6364 dev_info(&vsi->back->pdev->dev, 6365 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 6366 vsi->back->hw.aq.asq_last_status, ch->seid); 6367 return -EINVAL; 6368 } 6369 6370 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6371 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 6372 6373 return 0; 6374} 6375 6376/** 6377 * i40e_channel_config_tx_ring - config TX ring associated with new channel 6378 * @pf: ptr to PF device 6379 * @vsi: the VSI being setup 6380 * @ch: ptr to channel structure 6381 * 6382 * Configure TX rings associated with channel (VSI) since queues are being 6383 * from parent VSI. 6384 **/ 6385static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 6386 struct i40e_vsi *vsi, 6387 struct i40e_channel *ch) 6388{ 6389 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 6390 int ret; 6391 int i; 6392 6393 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 6394 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6395 if (ch->enabled_tc & BIT(i)) 6396 bw_share[i] = 1; 6397 } 6398 6399 /* configure BW for new VSI */ 6400 ret = i40e_channel_config_bw(vsi, ch, bw_share); 6401 if (ret) { 6402 dev_info(&vsi->back->pdev->dev, 6403 "Failed configuring TC map %d for channel (seid %u)\n", 6404 ch->enabled_tc, ch->seid); 6405 return ret; 6406 } 6407 6408 for (i = 0; i < ch->num_queue_pairs; i++) { 6409 struct i40e_ring *tx_ring, *rx_ring; 6410 u16 pf_q; 6411 6412 pf_q = ch->base_queue + i; 6413 6414 /* Get to TX ring ptr of main VSI, for re-setup TX queue 6415 * context 6416 */ 6417 tx_ring = vsi->tx_rings[pf_q]; 6418 tx_ring->ch = ch; 6419 6420 /* Get the RX ring ptr */ 6421 rx_ring = vsi->rx_rings[pf_q]; 6422 rx_ring->ch = ch; 6423 } 6424 6425 return 0; 6426} 6427 6428/** 6429 * i40e_setup_hw_channel - setup new channel 6430 * @pf: ptr to PF device 6431 * @vsi: the VSI being setup 6432 * @ch: ptr to channel structure 6433 * @uplink_seid: underlying HW switching element (VEB) ID 6434 * @type: type of channel to be created (VMDq2/VF) 6435 * 6436 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6437 * and configures TX rings accordingly 6438 **/ 6439static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6440 struct i40e_vsi *vsi, 6441 struct i40e_channel *ch, 6442 u16 uplink_seid, u8 type) 6443{ 6444 int ret; 6445 6446 ch->initialized = false; 6447 ch->base_queue = vsi->next_base_queue; 6448 ch->type = type; 6449 6450 /* Proceed with creation of channel (VMDq2) VSI */ 6451 ret = i40e_add_channel(pf, uplink_seid, ch); 6452 if (ret) { 6453 dev_info(&pf->pdev->dev, 6454 "failed to add_channel using uplink_seid %u\n", 6455 uplink_seid); 6456 return ret; 6457 } 6458 6459 /* Mark the successful creation of channel */ 6460 ch->initialized = true; 6461 6462 /* Reconfigure TX queues using QTX_CTL register */ 6463 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6464 if (ret) { 6465 dev_info(&pf->pdev->dev, 6466 "failed to configure TX rings for channel %u\n", 6467 ch->seid); 6468 return ret; 6469 } 6470 6471 /* update 'next_base_queue' */ 6472 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6473 dev_dbg(&pf->pdev->dev, 6474 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6475 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6476 ch->num_queue_pairs, 6477 vsi->next_base_queue); 6478 return ret; 6479} 6480 6481/** 6482 * i40e_setup_channel - setup new channel using uplink element 6483 * @pf: ptr to PF device 6484 * @vsi: pointer to the VSI to set up the channel within 6485 * @ch: ptr to channel structure 6486 * 6487 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6488 * and uplink switching element (uplink_seid) 6489 **/ 6490static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6491 struct i40e_channel *ch) 6492{ 6493 struct i40e_vsi *main_vsi; 6494 u8 vsi_type; 6495 u16 seid; 6496 int ret; 6497 6498 if (vsi->type == I40E_VSI_MAIN) { 6499 vsi_type = I40E_VSI_VMDQ2; 6500 } else { 6501 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6502 vsi->type); 6503 return false; 6504 } 6505 6506 /* underlying switching element */ 6507 main_vsi = i40e_pf_get_main_vsi(pf); 6508 seid = main_vsi->uplink_seid; 6509 6510 /* create channel (VSI), configure TX rings */ 6511 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6512 if (ret) { 6513 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6514 return false; 6515 } 6516 6517 return ch->initialized ? true : false; 6518} 6519 6520/** 6521 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6522 * @vsi: ptr to VSI which has PF backing 6523 * 6524 * Sets up switch mode correctly if it needs to be changed and perform 6525 * what are allowed modes. 6526 **/ 6527static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6528{ 6529 u8 mode; 6530 struct i40e_pf *pf = vsi->back; 6531 struct i40e_hw *hw = &pf->hw; 6532 int ret; 6533 6534 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6535 if (ret) 6536 return -EINVAL; 6537 6538 if (hw->dev_caps.switch_mode) { 6539 /* if switch mode is set, support mode2 (non-tunneled for 6540 * cloud filter) for now 6541 */ 6542 u32 switch_mode = hw->dev_caps.switch_mode & 6543 I40E_SWITCH_MODE_MASK; 6544 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6545 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6546 return 0; 6547 dev_err(&pf->pdev->dev, 6548 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6549 hw->dev_caps.switch_mode); 6550 return -EINVAL; 6551 } 6552 } 6553 6554 /* Set Bit 7 to be valid */ 6555 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6556 6557 /* Set L4type for TCP support */ 6558 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6559 6560 /* Set cloud filter mode */ 6561 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6562 6563 /* Prep mode field for set_switch_config */ 6564 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6565 pf->last_sw_conf_valid_flags, 6566 mode, NULL); 6567 if (ret && hw->aq.asq_last_status != LIBIE_AQ_RC_ESRCH) 6568 dev_err(&pf->pdev->dev, 6569 "couldn't set switch config bits, err %pe aq_err %s\n", 6570 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 6571 6572 return ret; 6573} 6574 6575/** 6576 * i40e_create_queue_channel - function to create channel 6577 * @vsi: VSI to be configured 6578 * @ch: ptr to channel (it contains channel specific params) 6579 * 6580 * This function creates channel (VSI) using num_queues specified by user, 6581 * reconfigs RSS if needed. 6582 **/ 6583int i40e_create_queue_channel(struct i40e_vsi *vsi, 6584 struct i40e_channel *ch) 6585{ 6586 struct i40e_pf *pf = vsi->back; 6587 bool reconfig_rss; 6588 int err; 6589 6590 if (!ch) 6591 return -EINVAL; 6592 6593 if (!ch->num_queue_pairs) { 6594 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6595 ch->num_queue_pairs); 6596 return -EINVAL; 6597 } 6598 6599 /* validate user requested num_queues for channel */ 6600 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6601 &reconfig_rss); 6602 if (err) { 6603 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6604 ch->num_queue_pairs); 6605 return -EINVAL; 6606 } 6607 6608 /* By default we are in VEPA mode, if this is the first VF/VMDq 6609 * VSI to be added switch to VEB mode. 6610 */ 6611 6612 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 6613 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 6614 6615 if (vsi->type == I40E_VSI_MAIN) { 6616 if (i40e_is_tc_mqprio_enabled(pf)) 6617 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 6618 else 6619 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); 6620 } 6621 /* now onwards for main VSI, number of queues will be value 6622 * of TC0's queue count 6623 */ 6624 } 6625 6626 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6627 * it should be more than num_queues 6628 */ 6629 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6630 dev_dbg(&pf->pdev->dev, 6631 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6632 vsi->cnt_q_avail, ch->num_queue_pairs); 6633 return -EINVAL; 6634 } 6635 6636 /* reconfig_rss only if vsi type is MAIN_VSI */ 6637 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6638 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6639 if (err) { 6640 dev_info(&pf->pdev->dev, 6641 "Error: unable to reconfig rss for num_queues (%u)\n", 6642 ch->num_queue_pairs); 6643 return -EINVAL; 6644 } 6645 } 6646 6647 if (!i40e_setup_channel(pf, vsi, ch)) { 6648 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6649 return -EINVAL; 6650 } 6651 6652 dev_info(&pf->pdev->dev, 6653 "Setup channel (id:%u) utilizing num_queues %d\n", 6654 ch->seid, ch->num_queue_pairs); 6655 6656 /* configure VSI for BW limit */ 6657 if (ch->max_tx_rate) { 6658 u64 credits = ch->max_tx_rate; 6659 6660 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6661 return -EINVAL; 6662 6663 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6664 dev_dbg(&pf->pdev->dev, 6665 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6666 ch->max_tx_rate, 6667 credits, 6668 ch->seid); 6669 } 6670 6671 /* in case of VF, this will be main SRIOV VSI */ 6672 ch->parent_vsi = vsi; 6673 6674 /* and update main_vsi's count for queue_available to use */ 6675 vsi->cnt_q_avail -= ch->num_queue_pairs; 6676 6677 return 0; 6678} 6679 6680/** 6681 * i40e_configure_queue_channels - Add queue channel for the given TCs 6682 * @vsi: VSI to be configured 6683 * 6684 * Configures queue channel mapping to the given TCs 6685 **/ 6686static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6687{ 6688 struct i40e_channel *ch; 6689 u64 max_rate = 0; 6690 int ret = 0, i; 6691 6692 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6693 vsi->tc_seid_map[0] = vsi->seid; 6694 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6695 if (vsi->tc_config.enabled_tc & BIT(i)) { 6696 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6697 if (!ch) { 6698 ret = -ENOMEM; 6699 goto err_free; 6700 } 6701 6702 INIT_LIST_HEAD(&ch->list); 6703 ch->num_queue_pairs = 6704 vsi->tc_config.tc_info[i].qcount; 6705 ch->base_queue = 6706 vsi->tc_config.tc_info[i].qoffset; 6707 6708 /* Bandwidth limit through tc interface is in bytes/s, 6709 * change to Mbit/s 6710 */ 6711 max_rate = vsi->mqprio_qopt.max_rate[i]; 6712 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6713 ch->max_tx_rate = max_rate; 6714 6715 list_add_tail(&ch->list, &vsi->ch_list); 6716 6717 ret = i40e_create_queue_channel(vsi, ch); 6718 if (ret) { 6719 dev_err(&vsi->back->pdev->dev, 6720 "Failed creating queue channel with TC%d: queues %d\n", 6721 i, ch->num_queue_pairs); 6722 goto err_free; 6723 } 6724 vsi->tc_seid_map[i] = ch->seid; 6725 } 6726 } 6727 6728 /* reset to reconfigure TX queue contexts */ 6729 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true); 6730 return ret; 6731 6732err_free: 6733 i40e_remove_queue_channels(vsi); 6734 return ret; 6735} 6736 6737/** 6738 * i40e_veb_config_tc - Configure TCs for given VEB 6739 * @veb: given VEB 6740 * @enabled_tc: TC bitmap 6741 * 6742 * Configures given TC bitmap for VEB (switching) element 6743 **/ 6744int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6745{ 6746 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6747 struct i40e_pf *pf = veb->pf; 6748 int ret = 0; 6749 int i; 6750 6751 /* No TCs or already enabled TCs just return */ 6752 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6753 return ret; 6754 6755 bw_data.tc_valid_bits = enabled_tc; 6756 /* bw_data.absolute_credits is not set (relative) */ 6757 6758 /* Enable ETS TCs with equal BW Share for now */ 6759 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6760 if (enabled_tc & BIT(i)) 6761 bw_data.tc_bw_share_credits[i] = 1; 6762 } 6763 6764 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6765 &bw_data, NULL); 6766 if (ret) { 6767 dev_info(&pf->pdev->dev, 6768 "VEB bw config failed, err %pe aq_err %s\n", 6769 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6770 goto out; 6771 } 6772 6773 /* Update the BW information */ 6774 ret = i40e_veb_get_bw_info(veb); 6775 if (ret) { 6776 dev_info(&pf->pdev->dev, 6777 "Failed getting veb bw config, err %pe aq_err %s\n", 6778 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6779 } 6780 6781out: 6782 return ret; 6783} 6784 6785#ifdef CONFIG_I40E_DCB 6786/** 6787 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6788 * @pf: PF struct 6789 * 6790 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6791 * the caller would've quiesce all the VSIs before calling 6792 * this function 6793 **/ 6794static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6795{ 6796 struct i40e_vsi *vsi; 6797 struct i40e_veb *veb; 6798 u8 tc_map = 0; 6799 int ret; 6800 int v; 6801 6802 /* Enable the TCs available on PF to all VEBs */ 6803 tc_map = i40e_pf_get_tc_map(pf); 6804 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS) 6805 return; 6806 6807 i40e_pf_for_each_veb(pf, v, veb) { 6808 ret = i40e_veb_config_tc(veb, tc_map); 6809 if (ret) { 6810 dev_info(&pf->pdev->dev, 6811 "Failed configuring TC for VEB seid=%d\n", 6812 veb->seid); 6813 /* Will try to configure as many components */ 6814 } 6815 } 6816 6817 /* Update each VSI */ 6818 i40e_pf_for_each_vsi(pf, v, vsi) { 6819 /* - Enable all TCs for the LAN VSI 6820 * - For all others keep them at TC0 for now 6821 */ 6822 if (vsi->type == I40E_VSI_MAIN) 6823 tc_map = i40e_pf_get_tc_map(pf); 6824 else 6825 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6826 6827 ret = i40e_vsi_config_tc(vsi, tc_map); 6828 if (ret) { 6829 dev_info(&pf->pdev->dev, 6830 "Failed configuring TC for VSI seid=%d\n", 6831 vsi->seid); 6832 /* Will try to configure as many components */ 6833 } else { 6834 /* Re-configure VSI vectors based on updated TC map */ 6835 i40e_vsi_map_rings_to_vectors(vsi); 6836 if (vsi->netdev) 6837 i40e_dcbnl_set_all(vsi); 6838 } 6839 } 6840} 6841 6842/** 6843 * i40e_resume_port_tx - Resume port Tx 6844 * @pf: PF struct 6845 * 6846 * Resume a port's Tx and issue a PF reset in case of failure to 6847 * resume. 6848 **/ 6849static int i40e_resume_port_tx(struct i40e_pf *pf) 6850{ 6851 struct i40e_hw *hw = &pf->hw; 6852 int ret; 6853 6854 ret = i40e_aq_resume_port_tx(hw, NULL); 6855 if (ret) { 6856 dev_info(&pf->pdev->dev, 6857 "Resume Port Tx failed, err %pe aq_err %s\n", 6858 ERR_PTR(ret), 6859 libie_aq_str(pf->hw.aq.asq_last_status)); 6860 /* Schedule PF reset to recover */ 6861 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6862 i40e_service_event_schedule(pf); 6863 } 6864 6865 return ret; 6866} 6867 6868/** 6869 * i40e_suspend_port_tx - Suspend port Tx 6870 * @pf: PF struct 6871 * 6872 * Suspend a port's Tx and issue a PF reset in case of failure. 6873 **/ 6874static int i40e_suspend_port_tx(struct i40e_pf *pf) 6875{ 6876 struct i40e_hw *hw = &pf->hw; 6877 int ret; 6878 6879 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL); 6880 if (ret) { 6881 dev_info(&pf->pdev->dev, 6882 "Suspend Port Tx failed, err %pe aq_err %s\n", 6883 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6884 /* Schedule PF reset to recover */ 6885 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6886 i40e_service_event_schedule(pf); 6887 } 6888 6889 return ret; 6890} 6891 6892/** 6893 * i40e_hw_set_dcb_config - Program new DCBX settings into HW 6894 * @pf: PF being configured 6895 * @new_cfg: New DCBX configuration 6896 * 6897 * Program DCB settings into HW and reconfigure VEB/VSIs on 6898 * given PF. Uses "Set LLDP MIB" AQC to program the hardware. 6899 **/ 6900static int i40e_hw_set_dcb_config(struct i40e_pf *pf, 6901 struct i40e_dcbx_config *new_cfg) 6902{ 6903 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config; 6904 int ret; 6905 6906 /* Check if need reconfiguration */ 6907 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) { 6908 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n"); 6909 return 0; 6910 } 6911 6912 /* Config change disable all VSIs */ 6913 i40e_pf_quiesce_all_vsi(pf); 6914 6915 /* Copy the new config to the current config */ 6916 *old_cfg = *new_cfg; 6917 old_cfg->etsrec = old_cfg->etscfg; 6918 ret = i40e_set_dcb_config(&pf->hw); 6919 if (ret) { 6920 dev_info(&pf->pdev->dev, 6921 "Set DCB Config failed, err %pe aq_err %s\n", 6922 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 6923 goto out; 6924 } 6925 6926 /* Changes in configuration update VEB/VSI */ 6927 i40e_dcb_reconfigure(pf); 6928out: 6929 /* In case of reset do not try to resume anything */ 6930 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) { 6931 /* Re-start the VSIs if disabled */ 6932 ret = i40e_resume_port_tx(pf); 6933 /* In case of error no point in resuming VSIs */ 6934 if (ret) 6935 goto err; 6936 i40e_pf_unquiesce_all_vsi(pf); 6937 } 6938err: 6939 return ret; 6940} 6941 6942/** 6943 * i40e_hw_dcb_config - Program new DCBX settings into HW 6944 * @pf: PF being configured 6945 * @new_cfg: New DCBX configuration 6946 * 6947 * Program DCB settings into HW and reconfigure VEB/VSIs on 6948 * given PF 6949 **/ 6950int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg) 6951{ 6952 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 6953 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0}; 6954 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS]; 6955 struct i40e_dcbx_config *old_cfg; 6956 u8 mode[I40E_MAX_TRAFFIC_CLASS]; 6957 struct i40e_rx_pb_config pb_cfg; 6958 struct i40e_hw *hw = &pf->hw; 6959 u8 num_ports = hw->num_ports; 6960 bool need_reconfig; 6961 int ret = -EINVAL; 6962 u8 lltc_map = 0; 6963 u8 tc_map = 0; 6964 u8 new_numtc; 6965 u8 i; 6966 6967 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n"); 6968 /* Un-pack information to Program ETS HW via shared API 6969 * numtc, tcmap 6970 * LLTC map 6971 * ETS/NON-ETS arbiter mode 6972 * max exponent (credit refills) 6973 * Total number of ports 6974 * PFC priority bit-map 6975 * Priority Table 6976 * BW % per TC 6977 * Arbiter mode between UPs sharing same TC 6978 * TSA table (ETS or non-ETS) 6979 * EEE enabled or not 6980 * MFS TC table 6981 */ 6982 6983 new_numtc = i40e_dcb_get_num_tc(new_cfg); 6984 6985 memset(&ets_data, 0, sizeof(ets_data)); 6986 for (i = 0; i < new_numtc; i++) { 6987 tc_map |= BIT(i); 6988 switch (new_cfg->etscfg.tsatable[i]) { 6989 case I40E_IEEE_TSA_ETS: 6990 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS; 6991 ets_data.tc_bw_share_credits[i] = 6992 new_cfg->etscfg.tcbwtable[i]; 6993 break; 6994 case I40E_IEEE_TSA_STRICT: 6995 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT; 6996 lltc_map |= BIT(i); 6997 ets_data.tc_bw_share_credits[i] = 6998 I40E_DCB_STRICT_PRIO_CREDITS; 6999 break; 7000 default: 7001 /* Invalid TSA type */ 7002 need_reconfig = false; 7003 goto out; 7004 } 7005 } 7006 7007 old_cfg = &hw->local_dcbx_config; 7008 /* Check if need reconfiguration */ 7009 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg); 7010 7011 /* If needed, enable/disable frame tagging, disable all VSIs 7012 * and suspend port tx 7013 */ 7014 if (need_reconfig) { 7015 /* Enable DCB tagging only when more than one TC */ 7016 if (new_numtc > 1) 7017 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7018 else 7019 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7020 7021 set_bit(__I40E_PORT_SUSPENDED, pf->state); 7022 /* Reconfiguration needed quiesce all VSIs */ 7023 i40e_pf_quiesce_all_vsi(pf); 7024 ret = i40e_suspend_port_tx(pf); 7025 if (ret) 7026 goto err; 7027 } 7028 7029 /* Configure Port ETS Tx Scheduler */ 7030 ets_data.tc_valid_bits = tc_map; 7031 ets_data.tc_strict_priority_flags = lltc_map; 7032 ret = i40e_aq_config_switch_comp_ets 7033 (hw, pf->mac_seid, &ets_data, 7034 i40e_aqc_opc_modify_switching_comp_ets, NULL); 7035 if (ret) { 7036 dev_info(&pf->pdev->dev, 7037 "Modify Port ETS failed, err %pe aq_err %s\n", 7038 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 7039 goto out; 7040 } 7041 7042 /* Configure Rx ETS HW */ 7043 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode)); 7044 i40e_dcb_hw_set_num_tc(hw, new_numtc); 7045 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN, 7046 I40E_DCB_ARB_MODE_STRICT_PRIORITY, 7047 I40E_DCB_DEFAULT_MAX_EXPONENT, 7048 lltc_map); 7049 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports); 7050 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode, 7051 prio_type); 7052 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable, 7053 new_cfg->etscfg.prioritytable); 7054 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable); 7055 7056 /* Configure Rx Packet Buffers in HW */ 7057 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7058 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 7059 7060 mfs_tc[i] = main_vsi->netdev->mtu; 7061 mfs_tc[i] += I40E_PACKET_HDR_PAD; 7062 } 7063 7064 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports, 7065 false, new_cfg->pfc.pfcenable, 7066 mfs_tc, &pb_cfg); 7067 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg); 7068 7069 /* Update the local Rx Packet buffer config */ 7070 pf->pb_cfg = pb_cfg; 7071 7072 /* Inform the FW about changes to DCB configuration */ 7073 ret = i40e_aq_dcb_updated(&pf->hw, NULL); 7074 if (ret) { 7075 dev_info(&pf->pdev->dev, 7076 "DCB Updated failed, err %pe aq_err %s\n", 7077 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 7078 goto out; 7079 } 7080 7081 /* Update the port DCBx configuration */ 7082 *old_cfg = *new_cfg; 7083 7084 /* Changes in configuration update VEB/VSI */ 7085 i40e_dcb_reconfigure(pf); 7086out: 7087 /* Re-start the VSIs if disabled */ 7088 if (need_reconfig) { 7089 ret = i40e_resume_port_tx(pf); 7090 7091 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 7092 /* In case of error no point in resuming VSIs */ 7093 if (ret) 7094 goto err; 7095 7096 /* Wait for the PF's queues to be disabled */ 7097 ret = i40e_pf_wait_queues_disabled(pf); 7098 if (ret) { 7099 /* Schedule PF reset to recover */ 7100 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7101 i40e_service_event_schedule(pf); 7102 goto err; 7103 } else { 7104 i40e_pf_unquiesce_all_vsi(pf); 7105 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7106 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 7107 } 7108 /* registers are set, lets apply */ 7109 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) 7110 ret = i40e_hw_set_dcb_config(pf, new_cfg); 7111 } 7112 7113err: 7114 return ret; 7115} 7116 7117/** 7118 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW 7119 * @pf: PF being queried 7120 * 7121 * Set default DCB configuration in case DCB is to be done in SW. 7122 **/ 7123int i40e_dcb_sw_default_config(struct i40e_pf *pf) 7124{ 7125 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config; 7126 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 7127 struct i40e_hw *hw = &pf->hw; 7128 int err; 7129 7130 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) { 7131 /* Update the local cached instance with TC0 ETS */ 7132 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config)); 7133 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7134 pf->tmp_cfg.etscfg.maxtcs = 0; 7135 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7136 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS; 7137 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING; 7138 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS; 7139 /* FW needs one App to configure HW */ 7140 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS; 7141 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE; 7142 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO; 7143 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE; 7144 7145 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg); 7146 } 7147 7148 memset(&ets_data, 0, sizeof(ets_data)); 7149 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */ 7150 ets_data.tc_strict_priority_flags = 0; /* ETS */ 7151 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */ 7152 7153 /* Enable ETS on the Physical port */ 7154 err = i40e_aq_config_switch_comp_ets 7155 (hw, pf->mac_seid, &ets_data, 7156 i40e_aqc_opc_enable_switching_comp_ets, NULL); 7157 if (err) { 7158 dev_info(&pf->pdev->dev, 7159 "Enable Port ETS failed, err %pe aq_err %s\n", 7160 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7161 err = -ENOENT; 7162 goto out; 7163 } 7164 7165 /* Update the local cached instance with TC0 ETS */ 7166 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7167 dcb_cfg->etscfg.cbs = 0; 7168 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS; 7169 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7170 7171out: 7172 return err; 7173} 7174 7175/** 7176 * i40e_init_pf_dcb - Initialize DCB configuration 7177 * @pf: PF being configured 7178 * 7179 * Query the current DCB configuration and cache it 7180 * in the hardware structure 7181 **/ 7182static int i40e_init_pf_dcb(struct i40e_pf *pf) 7183{ 7184 struct i40e_hw *hw = &pf->hw; 7185 int err; 7186 7187 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 7188 * Also do not enable DCBx if FW LLDP agent is disabled 7189 */ 7190 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) { 7191 dev_info(&pf->pdev->dev, "DCB is not supported.\n"); 7192 err = -EOPNOTSUPP; 7193 goto out; 7194 } 7195 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) { 7196 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n"); 7197 err = i40e_dcb_sw_default_config(pf); 7198 if (err) { 7199 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n"); 7200 goto out; 7201 } 7202 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n"); 7203 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 7204 DCB_CAP_DCBX_VER_IEEE; 7205 /* at init capable but disabled */ 7206 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7207 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7208 goto out; 7209 } 7210 err = i40e_init_dcb(hw, true); 7211 if (!err) { 7212 /* Device/Function is not DCBX capable */ 7213 if ((!hw->func_caps.dcb) || 7214 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 7215 dev_info(&pf->pdev->dev, 7216 "DCBX offload is not supported or is disabled for this PF.\n"); 7217 } else { 7218 /* When status is not DISABLED then DCBX in FW */ 7219 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 7220 DCB_CAP_DCBX_VER_IEEE; 7221 7222 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 7223 /* Enable DCB tagging only when more than one TC 7224 * or explicitly disable if only one TC 7225 */ 7226 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 7227 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 7228 else 7229 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 7230 dev_dbg(&pf->pdev->dev, 7231 "DCBX offload is supported for this PF.\n"); 7232 } 7233 } else if (pf->hw.aq.asq_last_status == LIBIE_AQ_RC_EPERM) { 7234 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 7235 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags); 7236 } else { 7237 dev_info(&pf->pdev->dev, 7238 "Query for DCB configuration failed, err %pe aq_err %s\n", 7239 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7240 } 7241 7242out: 7243 return err; 7244} 7245#endif /* CONFIG_I40E_DCB */ 7246 7247static void i40e_print_link_message_eee(struct i40e_vsi *vsi, 7248 const char *speed, const char *fc) 7249{ 7250 struct ethtool_keee kedata; 7251 7252 memzero_explicit(&kedata, sizeof(kedata)); 7253 if (vsi->netdev->ethtool_ops->get_eee) 7254 vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata); 7255 7256 if (!linkmode_empty(kedata.supported)) 7257 netdev_info(vsi->netdev, 7258 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n", 7259 speed, fc, 7260 kedata.eee_enabled ? "Enabled" : "Disabled"); 7261 else 7262 netdev_info(vsi->netdev, 7263 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 7264 speed, fc); 7265} 7266 7267/** 7268 * i40e_print_link_message - print link up or down 7269 * @vsi: the VSI for which link needs a message 7270 * @isup: true of link is up, false otherwise 7271 */ 7272void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 7273{ 7274 enum i40e_aq_link_speed new_speed; 7275 struct i40e_pf *pf = vsi->back; 7276 char *speed = "Unknown"; 7277 char *fc = "Unknown"; 7278 char *fec = ""; 7279 char *req_fec = ""; 7280 char *an = ""; 7281 7282 if (isup) 7283 new_speed = pf->hw.phy.link_info.link_speed; 7284 else 7285 new_speed = I40E_LINK_SPEED_UNKNOWN; 7286 7287 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 7288 return; 7289 vsi->current_isup = isup; 7290 vsi->current_speed = new_speed; 7291 if (!isup) { 7292 netdev_info(vsi->netdev, "NIC Link is Down\n"); 7293 return; 7294 } 7295 7296 /* Warn user if link speed on NPAR enabled partition is not at 7297 * least 10GB 7298 */ 7299 if (pf->hw.func_caps.npar_enable && 7300 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 7301 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 7302 netdev_warn(vsi->netdev, 7303 "The partition detected link speed that is less than 10Gbps\n"); 7304 7305 switch (pf->hw.phy.link_info.link_speed) { 7306 case I40E_LINK_SPEED_40GB: 7307 speed = "40 G"; 7308 break; 7309 case I40E_LINK_SPEED_20GB: 7310 speed = "20 G"; 7311 break; 7312 case I40E_LINK_SPEED_25GB: 7313 speed = "25 G"; 7314 break; 7315 case I40E_LINK_SPEED_10GB: 7316 speed = "10 G"; 7317 break; 7318 case I40E_LINK_SPEED_5GB: 7319 speed = "5 G"; 7320 break; 7321 case I40E_LINK_SPEED_2_5GB: 7322 speed = "2.5 G"; 7323 break; 7324 case I40E_LINK_SPEED_1GB: 7325 speed = "1000 M"; 7326 break; 7327 case I40E_LINK_SPEED_100MB: 7328 speed = "100 M"; 7329 break; 7330 default: 7331 break; 7332 } 7333 7334 switch (pf->hw.fc.current_mode) { 7335 case I40E_FC_FULL: 7336 fc = "RX/TX"; 7337 break; 7338 case I40E_FC_TX_PAUSE: 7339 fc = "TX"; 7340 break; 7341 case I40E_FC_RX_PAUSE: 7342 fc = "RX"; 7343 break; 7344 default: 7345 fc = "None"; 7346 break; 7347 } 7348 7349 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 7350 req_fec = "None"; 7351 fec = "None"; 7352 an = "False"; 7353 7354 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7355 an = "True"; 7356 7357 if (pf->hw.phy.link_info.fec_info & 7358 I40E_AQ_CONFIG_FEC_KR_ENA) 7359 fec = "CL74 FC-FEC/BASE-R"; 7360 else if (pf->hw.phy.link_info.fec_info & 7361 I40E_AQ_CONFIG_FEC_RS_ENA) 7362 fec = "CL108 RS-FEC"; 7363 7364 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 7365 * both RS and FC are requested 7366 */ 7367 if (vsi->back->hw.phy.link_info.req_fec_info & 7368 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 7369 if (vsi->back->hw.phy.link_info.req_fec_info & 7370 I40E_AQ_REQUEST_FEC_RS) 7371 req_fec = "CL108 RS-FEC"; 7372 else 7373 req_fec = "CL74 FC-FEC/BASE-R"; 7374 } 7375 netdev_info(vsi->netdev, 7376 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7377 speed, req_fec, fec, an, fc); 7378 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) { 7379 req_fec = "None"; 7380 fec = "None"; 7381 an = "False"; 7382 7383 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7384 an = "True"; 7385 7386 if (pf->hw.phy.link_info.fec_info & 7387 I40E_AQ_CONFIG_FEC_KR_ENA) 7388 fec = "CL74 FC-FEC/BASE-R"; 7389 7390 if (pf->hw.phy.link_info.req_fec_info & 7391 I40E_AQ_REQUEST_FEC_KR) 7392 req_fec = "CL74 FC-FEC/BASE-R"; 7393 7394 netdev_info(vsi->netdev, 7395 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7396 speed, req_fec, fec, an, fc); 7397 } else { 7398 i40e_print_link_message_eee(vsi, speed, fc); 7399 } 7400 7401} 7402 7403/** 7404 * i40e_up_complete - Finish the last steps of bringing up a connection 7405 * @vsi: the VSI being configured 7406 **/ 7407static int i40e_up_complete(struct i40e_vsi *vsi) 7408{ 7409 struct i40e_pf *pf = vsi->back; 7410 int err; 7411 7412 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 7413 i40e_vsi_configure_msix(vsi); 7414 else 7415 i40e_configure_msi_and_legacy(vsi); 7416 7417 /* start rings */ 7418 err = i40e_vsi_start_rings(vsi); 7419 if (err) 7420 return err; 7421 7422 clear_bit(__I40E_VSI_DOWN, vsi->state); 7423 i40e_napi_enable_all(vsi); 7424 i40e_vsi_enable_irq(vsi); 7425 7426 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 7427 (vsi->netdev)) { 7428 i40e_print_link_message(vsi, true); 7429 netif_tx_start_all_queues(vsi->netdev); 7430 netif_carrier_on(vsi->netdev); 7431 } 7432 7433 /* replay FDIR SB filters */ 7434 if (vsi->type == I40E_VSI_FDIR) { 7435 /* reset fd counters */ 7436 pf->fd_add_err = 0; 7437 pf->fd_atr_cnt = 0; 7438 i40e_fdir_filter_restore(vsi); 7439 } 7440 7441 /* On the next run of the service_task, notify any clients of the new 7442 * opened netdev 7443 */ 7444 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7445 i40e_service_event_schedule(pf); 7446 7447 return 0; 7448} 7449 7450/** 7451 * i40e_vsi_reinit_locked - Reset the VSI 7452 * @vsi: the VSI being configured 7453 * 7454 * Rebuild the ring structs after some configuration 7455 * has changed, e.g. MTU size. 7456 **/ 7457static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 7458{ 7459 struct i40e_pf *pf = vsi->back; 7460 7461 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 7462 usleep_range(1000, 2000); 7463 i40e_down(vsi); 7464 7465 i40e_up(vsi); 7466 clear_bit(__I40E_CONFIG_BUSY, pf->state); 7467} 7468 7469/** 7470 * i40e_force_link_state - Force the link status 7471 * @pf: board private structure 7472 * @is_up: whether the link state should be forced up or down 7473 **/ 7474static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) 7475{ 7476 struct i40e_aq_get_phy_abilities_resp abilities; 7477 struct i40e_aq_set_phy_config config = {0}; 7478 bool non_zero_phy_type = is_up; 7479 struct i40e_hw *hw = &pf->hw; 7480 u64 mask; 7481 u8 speed; 7482 int err; 7483 7484 /* Card might've been put in an unstable state by other drivers 7485 * and applications, which causes incorrect speed values being 7486 * set on startup. In order to clear speed registers, we call 7487 * get_phy_capabilities twice, once to get initial state of 7488 * available speeds, and once to get current PHY config. 7489 */ 7490 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 7491 NULL); 7492 if (err) { 7493 dev_err(&pf->pdev->dev, 7494 "failed to get phy cap., ret = %pe last_status = %s\n", 7495 ERR_PTR(err), libie_aq_str(hw->aq.asq_last_status)); 7496 return err; 7497 } 7498 speed = abilities.link_speed; 7499 7500 /* Get the current phy config */ 7501 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 7502 NULL); 7503 if (err) { 7504 dev_err(&pf->pdev->dev, 7505 "failed to get phy cap., ret = %pe last_status = %s\n", 7506 ERR_PTR(err), libie_aq_str(hw->aq.asq_last_status)); 7507 return err; 7508 } 7509 7510 /* If link needs to go up, but was not forced to go down, 7511 * and its speed values are OK, no need for a flap 7512 * if non_zero_phy_type was set, still need to force up 7513 */ 7514 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) 7515 non_zero_phy_type = true; 7516 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 7517 return 0; 7518 7519 /* To force link we need to set bits for all supported PHY types, 7520 * but there are now more than 32, so we need to split the bitmap 7521 * across two fields. 7522 */ 7523 mask = I40E_PHY_TYPES_BITMASK; 7524 config.phy_type = 7525 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 7526 config.phy_type_ext = 7527 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 7528 /* Copy the old settings, except of phy_type */ 7529 config.abilities = abilities.abilities; 7530 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) { 7531 if (is_up) 7532 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 7533 else 7534 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 7535 } 7536 if (abilities.link_speed != 0) 7537 config.link_speed = abilities.link_speed; 7538 else 7539 config.link_speed = speed; 7540 config.eee_capability = abilities.eee_capability; 7541 config.eeer = abilities.eeer_val; 7542 config.low_power_ctrl = abilities.d3_lpan; 7543 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 7544 I40E_AQ_PHY_FEC_CONFIG_MASK; 7545 err = i40e_aq_set_phy_config(hw, &config, NULL); 7546 7547 if (err) { 7548 dev_err(&pf->pdev->dev, 7549 "set phy config ret = %pe last_status = %s\n", 7550 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 7551 return err; 7552 } 7553 7554 /* Update the link info */ 7555 err = i40e_update_link_info(hw); 7556 if (err) { 7557 /* Wait a little bit (on 40G cards it sometimes takes a really 7558 * long time for link to come back from the atomic reset) 7559 * and try once more 7560 */ 7561 msleep(1000); 7562 i40e_update_link_info(hw); 7563 } 7564 7565 i40e_aq_set_link_restart_an(hw, is_up, NULL); 7566 7567 return 0; 7568} 7569 7570/** 7571 * i40e_up - Bring the connection back up after being down 7572 * @vsi: the VSI being configured 7573 **/ 7574int i40e_up(struct i40e_vsi *vsi) 7575{ 7576 int err; 7577 7578 if (vsi->type == I40E_VSI_MAIN && 7579 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7580 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7581 i40e_force_link_state(vsi->back, true); 7582 7583 err = i40e_vsi_configure(vsi); 7584 if (!err) 7585 err = i40e_up_complete(vsi); 7586 7587 return err; 7588} 7589 7590/** 7591 * i40e_down - Shutdown the connection processing 7592 * @vsi: the VSI being stopped 7593 **/ 7594void i40e_down(struct i40e_vsi *vsi) 7595{ 7596 int i; 7597 7598 /* It is assumed that the caller of this function 7599 * sets the vsi->state __I40E_VSI_DOWN bit. 7600 */ 7601 if (vsi->netdev) { 7602 netif_carrier_off(vsi->netdev); 7603 netif_tx_disable(vsi->netdev); 7604 } 7605 i40e_vsi_disable_irq(vsi); 7606 i40e_vsi_stop_rings(vsi); 7607 if (vsi->type == I40E_VSI_MAIN && 7608 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) || 7609 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags))) 7610 i40e_force_link_state(vsi->back, false); 7611 i40e_napi_disable_all(vsi); 7612 7613 for (i = 0; i < vsi->num_queue_pairs; i++) { 7614 i40e_clean_tx_ring(vsi->tx_rings[i]); 7615 if (i40e_enabled_xdp_vsi(vsi)) { 7616 /* Make sure that in-progress ndo_xdp_xmit and 7617 * ndo_xsk_wakeup calls are completed. 7618 */ 7619 synchronize_rcu(); 7620 i40e_clean_tx_ring(vsi->xdp_rings[i]); 7621 } 7622 i40e_clean_rx_ring(vsi->rx_rings[i]); 7623 } 7624 7625} 7626 7627/** 7628 * i40e_validate_mqprio_qopt- validate queue mapping info 7629 * @vsi: the VSI being configured 7630 * @mqprio_qopt: queue parametrs 7631 **/ 7632static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 7633 struct tc_mqprio_qopt_offload *mqprio_qopt) 7634{ 7635 u64 sum_max_rate = 0; 7636 u64 max_rate = 0; 7637 int i; 7638 7639 if (mqprio_qopt->qopt.offset[0] != 0 || 7640 mqprio_qopt->qopt.num_tc < 1 || 7641 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 7642 return -EINVAL; 7643 for (i = 0; ; i++) { 7644 if (!mqprio_qopt->qopt.count[i]) 7645 return -EINVAL; 7646 if (mqprio_qopt->min_rate[i]) { 7647 dev_err(&vsi->back->pdev->dev, 7648 "Invalid min tx rate (greater than 0) specified\n"); 7649 return -EINVAL; 7650 } 7651 max_rate = mqprio_qopt->max_rate[i]; 7652 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 7653 sum_max_rate += max_rate; 7654 7655 if (i >= mqprio_qopt->qopt.num_tc - 1) 7656 break; 7657 if (mqprio_qopt->qopt.offset[i + 1] != 7658 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7659 return -EINVAL; 7660 } 7661 if (vsi->num_queue_pairs < 7662 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 7663 dev_err(&vsi->back->pdev->dev, 7664 "Failed to create traffic channel, insufficient number of queues.\n"); 7665 return -EINVAL; 7666 } 7667 if (sum_max_rate > i40e_get_link_speed(vsi)) { 7668 dev_err(&vsi->back->pdev->dev, 7669 "Invalid max tx rate specified\n"); 7670 return -EINVAL; 7671 } 7672 return 0; 7673} 7674 7675/** 7676 * i40e_vsi_set_default_tc_config - set default values for tc configuration 7677 * @vsi: the VSI being configured 7678 **/ 7679static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 7680{ 7681 u16 qcount; 7682 int i; 7683 7684 /* Only TC0 is enabled */ 7685 vsi->tc_config.numtc = 1; 7686 vsi->tc_config.enabled_tc = 1; 7687 qcount = min_t(int, vsi->alloc_queue_pairs, 7688 i40e_pf_get_max_q_per_tc(vsi->back)); 7689 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7690 /* For the TC that is not enabled set the offset to default 7691 * queue and allocate one queue for the given TC. 7692 */ 7693 vsi->tc_config.tc_info[i].qoffset = 0; 7694 if (i == 0) 7695 vsi->tc_config.tc_info[i].qcount = qcount; 7696 else 7697 vsi->tc_config.tc_info[i].qcount = 1; 7698 vsi->tc_config.tc_info[i].netdev_tc = 0; 7699 } 7700} 7701 7702/** 7703 * i40e_del_macvlan_filter 7704 * @hw: pointer to the HW structure 7705 * @seid: seid of the channel VSI 7706 * @macaddr: the mac address to apply as a filter 7707 * @aq_err: store the admin Q error 7708 * 7709 * This function deletes a mac filter on the channel VSI which serves as the 7710 * macvlan. Returns 0 on success. 7711 **/ 7712static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 7713 const u8 *macaddr, int *aq_err) 7714{ 7715 struct i40e_aqc_remove_macvlan_element_data element; 7716 int status; 7717 7718 memset(&element, 0, sizeof(element)); 7719 ether_addr_copy(element.mac_addr, macaddr); 7720 element.vlan_tag = 0; 7721 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 7722 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 7723 *aq_err = hw->aq.asq_last_status; 7724 7725 return status; 7726} 7727 7728/** 7729 * i40e_add_macvlan_filter 7730 * @hw: pointer to the HW structure 7731 * @seid: seid of the channel VSI 7732 * @macaddr: the mac address to apply as a filter 7733 * @aq_err: store the admin Q error 7734 * 7735 * This function adds a mac filter on the channel VSI which serves as the 7736 * macvlan. Returns 0 on success. 7737 **/ 7738static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 7739 const u8 *macaddr, int *aq_err) 7740{ 7741 struct i40e_aqc_add_macvlan_element_data element; 7742 u16 cmd_flags = 0; 7743 int status; 7744 7745 ether_addr_copy(element.mac_addr, macaddr); 7746 element.vlan_tag = 0; 7747 element.queue_number = 0; 7748 element.match_method = I40E_AQC_MM_ERR_NO_RES; 7749 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 7750 element.flags = cpu_to_le16(cmd_flags); 7751 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 7752 *aq_err = hw->aq.asq_last_status; 7753 7754 return status; 7755} 7756 7757/** 7758 * i40e_reset_ch_rings - Reset the queue contexts in a channel 7759 * @vsi: the VSI we want to access 7760 * @ch: the channel we want to access 7761 */ 7762static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 7763{ 7764 struct i40e_ring *tx_ring, *rx_ring; 7765 u16 pf_q; 7766 int i; 7767 7768 for (i = 0; i < ch->num_queue_pairs; i++) { 7769 pf_q = ch->base_queue + i; 7770 tx_ring = vsi->tx_rings[pf_q]; 7771 tx_ring->ch = NULL; 7772 rx_ring = vsi->rx_rings[pf_q]; 7773 rx_ring->ch = NULL; 7774 } 7775} 7776 7777/** 7778 * i40e_free_macvlan_channels 7779 * @vsi: the VSI we want to access 7780 * 7781 * This function frees the Qs of the channel VSI from 7782 * the stack and also deletes the channel VSIs which 7783 * serve as macvlans. 7784 */ 7785static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7786{ 7787 struct i40e_channel *ch, *ch_tmp; 7788 int ret; 7789 7790 if (list_empty(&vsi->macvlan_list)) 7791 return; 7792 7793 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7794 struct i40e_vsi *parent_vsi; 7795 7796 if (i40e_is_channel_macvlan(ch)) { 7797 i40e_reset_ch_rings(vsi, ch); 7798 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7799 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7800 netdev_set_sb_channel(ch->fwd->netdev, 0); 7801 kfree(ch->fwd); 7802 ch->fwd = NULL; 7803 } 7804 7805 list_del(&ch->list); 7806 parent_vsi = ch->parent_vsi; 7807 if (!parent_vsi || !ch->initialized) { 7808 kfree(ch); 7809 continue; 7810 } 7811 7812 /* remove the VSI */ 7813 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7814 NULL); 7815 if (ret) 7816 dev_err(&vsi->back->pdev->dev, 7817 "unable to remove channel (%d) for parent VSI(%d)\n", 7818 ch->seid, parent_vsi->seid); 7819 kfree(ch); 7820 } 7821 vsi->macvlan_cnt = 0; 7822} 7823 7824/** 7825 * i40e_fwd_ring_up - bring the macvlan device up 7826 * @vsi: the VSI we want to access 7827 * @vdev: macvlan netdevice 7828 * @fwd: the private fwd structure 7829 */ 7830static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7831 struct i40e_fwd_adapter *fwd) 7832{ 7833 struct i40e_channel *ch = NULL, *ch_tmp, *iter; 7834 int ret = 0, num_tc = 1, i, aq_err; 7835 struct i40e_pf *pf = vsi->back; 7836 struct i40e_hw *hw = &pf->hw; 7837 7838 /* Go through the list and find an available channel */ 7839 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) { 7840 if (!i40e_is_channel_macvlan(iter)) { 7841 iter->fwd = fwd; 7842 /* record configuration for macvlan interface in vdev */ 7843 for (i = 0; i < num_tc; i++) 7844 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7845 i, 7846 iter->num_queue_pairs, 7847 iter->base_queue); 7848 for (i = 0; i < iter->num_queue_pairs; i++) { 7849 struct i40e_ring *tx_ring, *rx_ring; 7850 u16 pf_q; 7851 7852 pf_q = iter->base_queue + i; 7853 7854 /* Get to TX ring ptr */ 7855 tx_ring = vsi->tx_rings[pf_q]; 7856 tx_ring->ch = iter; 7857 7858 /* Get the RX ring ptr */ 7859 rx_ring = vsi->rx_rings[pf_q]; 7860 rx_ring->ch = iter; 7861 } 7862 ch = iter; 7863 break; 7864 } 7865 } 7866 7867 if (!ch) 7868 return -EINVAL; 7869 7870 /* Guarantee all rings are updated before we update the 7871 * MAC address filter. 7872 */ 7873 wmb(); 7874 7875 /* Add a mac filter */ 7876 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7877 if (ret) { 7878 /* if we cannot add the MAC rule then disable the offload */ 7879 macvlan_release_l2fw_offload(vdev); 7880 for (i = 0; i < ch->num_queue_pairs; i++) { 7881 struct i40e_ring *rx_ring; 7882 u16 pf_q; 7883 7884 pf_q = ch->base_queue + i; 7885 rx_ring = vsi->rx_rings[pf_q]; 7886 rx_ring->netdev = NULL; 7887 } 7888 dev_info(&pf->pdev->dev, 7889 "Error adding mac filter on macvlan err %pe, aq_err %s\n", 7890 ERR_PTR(ret), libie_aq_str(aq_err)); 7891 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7892 } 7893 7894 return ret; 7895} 7896 7897/** 7898 * i40e_setup_macvlans - create the channels which will be macvlans 7899 * @vsi: the VSI we want to access 7900 * @macvlan_cnt: no. of macvlans to be setup 7901 * @qcnt: no. of Qs per macvlan 7902 * @vdev: macvlan netdevice 7903 */ 7904static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7905 struct net_device *vdev) 7906{ 7907 struct i40e_pf *pf = vsi->back; 7908 struct i40e_hw *hw = &pf->hw; 7909 struct i40e_vsi_context ctxt; 7910 u16 sections, qmap, num_qps; 7911 struct i40e_channel *ch; 7912 int i, pow, ret = 0; 7913 u8 offset = 0; 7914 7915 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7916 return -EINVAL; 7917 7918 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7919 7920 /* find the next higher power-of-2 of num queue pairs */ 7921 pow = fls(roundup_pow_of_two(num_qps) - 1); 7922 7923 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7924 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7925 7926 /* Setup context bits for the main VSI */ 7927 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7928 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7929 memset(&ctxt, 0, sizeof(ctxt)); 7930 ctxt.seid = vsi->seid; 7931 ctxt.pf_num = vsi->back->hw.pf_id; 7932 ctxt.vf_num = 0; 7933 ctxt.uplink_seid = vsi->uplink_seid; 7934 ctxt.info = vsi->info; 7935 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7936 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7937 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7938 ctxt.info.valid_sections |= cpu_to_le16(sections); 7939 7940 /* Reconfigure RSS for main VSI with new max queue count */ 7941 vsi->rss_size = max_t(u16, num_qps, qcnt); 7942 ret = i40e_vsi_config_rss(vsi); 7943 if (ret) { 7944 dev_info(&pf->pdev->dev, 7945 "Failed to reconfig RSS for num_queues (%u)\n", 7946 vsi->rss_size); 7947 return ret; 7948 } 7949 vsi->reconfig_rss = true; 7950 dev_dbg(&vsi->back->pdev->dev, 7951 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7952 vsi->next_base_queue = num_qps; 7953 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7954 7955 /* Update the VSI after updating the VSI queue-mapping 7956 * information 7957 */ 7958 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7959 if (ret) { 7960 dev_info(&pf->pdev->dev, 7961 "Update vsi tc config failed, err %pe aq_err %s\n", 7962 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 7963 return ret; 7964 } 7965 /* update the local VSI info with updated queue map */ 7966 i40e_vsi_update_queue_map(vsi, &ctxt); 7967 vsi->info.valid_sections = 0; 7968 7969 /* Create channels for macvlans */ 7970 INIT_LIST_HEAD(&vsi->macvlan_list); 7971 for (i = 0; i < macvlan_cnt; i++) { 7972 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7973 if (!ch) { 7974 ret = -ENOMEM; 7975 goto err_free; 7976 } 7977 INIT_LIST_HEAD(&ch->list); 7978 ch->num_queue_pairs = qcnt; 7979 if (!i40e_setup_channel(pf, vsi, ch)) { 7980 ret = -EINVAL; 7981 kfree(ch); 7982 goto err_free; 7983 } 7984 ch->parent_vsi = vsi; 7985 vsi->cnt_q_avail -= ch->num_queue_pairs; 7986 vsi->macvlan_cnt++; 7987 list_add_tail(&ch->list, &vsi->macvlan_list); 7988 } 7989 7990 return ret; 7991 7992err_free: 7993 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7994 i40e_free_macvlan_channels(vsi); 7995 7996 return ret; 7997} 7998 7999/** 8000 * i40e_fwd_add - configure macvlans 8001 * @netdev: net device to configure 8002 * @vdev: macvlan netdevice 8003 **/ 8004static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 8005{ 8006 struct i40e_netdev_priv *np = netdev_priv(netdev); 8007 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 8008 struct i40e_vsi *vsi = np->vsi; 8009 struct i40e_pf *pf = vsi->back; 8010 struct i40e_fwd_adapter *fwd; 8011 int avail_macvlan, ret; 8012 8013 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8014 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 8015 return ERR_PTR(-EINVAL); 8016 } 8017 if (i40e_is_tc_mqprio_enabled(pf)) { 8018 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 8019 return ERR_PTR(-EINVAL); 8020 } 8021 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 8022 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 8023 return ERR_PTR(-EINVAL); 8024 } 8025 8026 /* The macvlan device has to be a single Q device so that the 8027 * tc_to_txq field can be reused to pick the tx queue. 8028 */ 8029 if (netif_is_multiqueue(vdev)) 8030 return ERR_PTR(-ERANGE); 8031 8032 if (!vsi->macvlan_cnt) { 8033 /* reserve bit 0 for the pf device */ 8034 set_bit(0, vsi->fwd_bitmask); 8035 8036 /* Try to reserve as many queues as possible for macvlans. First 8037 * reserve 3/4th of max vectors, then half, then quarter and 8038 * calculate Qs per macvlan as you go 8039 */ 8040 vectors = pf->num_lan_msix; 8041 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 8042 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 8043 q_per_macvlan = 4; 8044 macvlan_cnt = (vectors - 32) / 4; 8045 } else if (vectors <= 64 && vectors > 32) { 8046 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 8047 q_per_macvlan = 2; 8048 macvlan_cnt = (vectors - 16) / 2; 8049 } else if (vectors <= 32 && vectors > 16) { 8050 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 8051 q_per_macvlan = 1; 8052 macvlan_cnt = vectors - 16; 8053 } else if (vectors <= 16 && vectors > 8) { 8054 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 8055 q_per_macvlan = 1; 8056 macvlan_cnt = vectors - 8; 8057 } else { 8058 /* allocate 1 Q per macvlan and 1 Q to the PF */ 8059 q_per_macvlan = 1; 8060 macvlan_cnt = vectors - 1; 8061 } 8062 8063 if (macvlan_cnt == 0) 8064 return ERR_PTR(-EBUSY); 8065 8066 /* Quiesce VSI queues */ 8067 i40e_quiesce_vsi(vsi); 8068 8069 /* sets up the macvlans but does not "enable" them */ 8070 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 8071 vdev); 8072 if (ret) 8073 return ERR_PTR(ret); 8074 8075 /* Unquiesce VSI */ 8076 i40e_unquiesce_vsi(vsi); 8077 } 8078 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 8079 vsi->macvlan_cnt); 8080 if (avail_macvlan >= I40E_MAX_MACVLANS) 8081 return ERR_PTR(-EBUSY); 8082 8083 /* create the fwd struct */ 8084 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 8085 if (!fwd) 8086 return ERR_PTR(-ENOMEM); 8087 8088 set_bit(avail_macvlan, vsi->fwd_bitmask); 8089 fwd->bit_no = avail_macvlan; 8090 netdev_set_sb_channel(vdev, avail_macvlan); 8091 fwd->netdev = vdev; 8092 8093 if (!netif_running(netdev)) 8094 return fwd; 8095 8096 /* Set fwd ring up */ 8097 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 8098 if (ret) { 8099 /* unbind the queues and drop the subordinate channel config */ 8100 netdev_unbind_sb_channel(netdev, vdev); 8101 netdev_set_sb_channel(vdev, 0); 8102 8103 kfree(fwd); 8104 return ERR_PTR(-EINVAL); 8105 } 8106 8107 return fwd; 8108} 8109 8110/** 8111 * i40e_del_all_macvlans - Delete all the mac filters on the channels 8112 * @vsi: the VSI we want to access 8113 */ 8114static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 8115{ 8116 struct i40e_channel *ch, *ch_tmp; 8117 struct i40e_pf *pf = vsi->back; 8118 struct i40e_hw *hw = &pf->hw; 8119 int aq_err, ret = 0; 8120 8121 if (list_empty(&vsi->macvlan_list)) 8122 return; 8123 8124 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8125 if (i40e_is_channel_macvlan(ch)) { 8126 ret = i40e_del_macvlan_filter(hw, ch->seid, 8127 i40e_channel_mac(ch), 8128 &aq_err); 8129 if (!ret) { 8130 /* Reset queue contexts */ 8131 i40e_reset_ch_rings(vsi, ch); 8132 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8133 netdev_unbind_sb_channel(vsi->netdev, 8134 ch->fwd->netdev); 8135 netdev_set_sb_channel(ch->fwd->netdev, 0); 8136 kfree(ch->fwd); 8137 ch->fwd = NULL; 8138 } 8139 } 8140 } 8141} 8142 8143/** 8144 * i40e_fwd_del - delete macvlan interfaces 8145 * @netdev: net device to configure 8146 * @vdev: macvlan netdevice 8147 */ 8148static void i40e_fwd_del(struct net_device *netdev, void *vdev) 8149{ 8150 struct i40e_netdev_priv *np = netdev_priv(netdev); 8151 struct i40e_fwd_adapter *fwd = vdev; 8152 struct i40e_channel *ch, *ch_tmp; 8153 struct i40e_vsi *vsi = np->vsi; 8154 struct i40e_pf *pf = vsi->back; 8155 struct i40e_hw *hw = &pf->hw; 8156 int aq_err, ret = 0; 8157 8158 /* Find the channel associated with the macvlan and del mac filter */ 8159 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8160 if (i40e_is_channel_macvlan(ch) && 8161 ether_addr_equal(i40e_channel_mac(ch), 8162 fwd->netdev->dev_addr)) { 8163 ret = i40e_del_macvlan_filter(hw, ch->seid, 8164 i40e_channel_mac(ch), 8165 &aq_err); 8166 if (!ret) { 8167 /* Reset queue contexts */ 8168 i40e_reset_ch_rings(vsi, ch); 8169 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8170 netdev_unbind_sb_channel(netdev, fwd->netdev); 8171 netdev_set_sb_channel(fwd->netdev, 0); 8172 kfree(ch->fwd); 8173 ch->fwd = NULL; 8174 } else { 8175 dev_info(&pf->pdev->dev, 8176 "Error deleting mac filter on macvlan err %pe, aq_err %s\n", 8177 ERR_PTR(ret), libie_aq_str(aq_err)); 8178 } 8179 break; 8180 } 8181 } 8182} 8183 8184/** 8185 * i40e_setup_tc - configure multiple traffic classes 8186 * @netdev: net device to configure 8187 * @type_data: tc offload data 8188 **/ 8189static int i40e_setup_tc(struct net_device *netdev, void *type_data) 8190{ 8191 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8192 struct i40e_netdev_priv *np = netdev_priv(netdev); 8193 struct i40e_vsi *vsi = np->vsi; 8194 struct i40e_pf *pf = vsi->back; 8195 u8 enabled_tc = 0, num_tc, hw; 8196 bool need_reset = false; 8197 int old_queue_pairs; 8198 int ret = -EINVAL; 8199 u16 mode; 8200 int i; 8201 8202 old_queue_pairs = vsi->num_queue_pairs; 8203 num_tc = mqprio_qopt->qopt.num_tc; 8204 hw = mqprio_qopt->qopt.hw; 8205 mode = mqprio_qopt->mode; 8206 if (!hw) { 8207 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8208 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8209 goto config_tc; 8210 } 8211 8212 /* Check if MFP enabled */ 8213 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) { 8214 netdev_info(netdev, 8215 "Configuring TC not supported in MFP mode\n"); 8216 return ret; 8217 } 8218 switch (mode) { 8219 case TC_MQPRIO_MODE_DCB: 8220 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8221 8222 /* Check if DCB enabled to continue */ 8223 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8224 netdev_info(netdev, 8225 "DCB is not enabled for adapter\n"); 8226 return ret; 8227 } 8228 8229 /* Check whether tc count is within enabled limit */ 8230 if (num_tc > i40e_pf_get_num_tc(pf)) { 8231 netdev_info(netdev, 8232 "TC count greater than enabled on link for adapter\n"); 8233 return ret; 8234 } 8235 break; 8236 case TC_MQPRIO_MODE_CHANNEL: 8237 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) { 8238 netdev_info(netdev, 8239 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 8240 return ret; 8241 } 8242 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 8243 return ret; 8244 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 8245 if (ret) 8246 return ret; 8247 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 8248 sizeof(*mqprio_qopt)); 8249 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags); 8250 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 8251 break; 8252 default: 8253 return -EINVAL; 8254 } 8255 8256config_tc: 8257 /* Generate TC map for number of tc requested */ 8258 for (i = 0; i < num_tc; i++) 8259 enabled_tc |= BIT(i); 8260 8261 /* Requesting same TC configuration as already enabled */ 8262 if (enabled_tc == vsi->tc_config.enabled_tc && 8263 mode != TC_MQPRIO_MODE_CHANNEL) 8264 return 0; 8265 8266 /* Quiesce VSI queues */ 8267 i40e_quiesce_vsi(vsi); 8268 8269 if (!hw && !i40e_is_tc_mqprio_enabled(pf)) 8270 i40e_remove_queue_channels(vsi); 8271 8272 /* Configure VSI for enabled TCs */ 8273 ret = i40e_vsi_config_tc(vsi, enabled_tc); 8274 if (ret) { 8275 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 8276 vsi->seid); 8277 need_reset = true; 8278 goto exit; 8279 } else if (enabled_tc && 8280 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) { 8281 netdev_info(netdev, 8282 "Failed to create channel. Override queues (%u) not power of 2\n", 8283 vsi->tc_config.tc_info[0].qcount); 8284 ret = -EINVAL; 8285 need_reset = true; 8286 goto exit; 8287 } 8288 8289 dev_info(&vsi->back->pdev->dev, 8290 "Setup channel (id:%u) utilizing num_queues %d\n", 8291 vsi->seid, vsi->tc_config.tc_info[0].qcount); 8292 8293 if (i40e_is_tc_mqprio_enabled(pf)) { 8294 if (vsi->mqprio_qopt.max_rate[0]) { 8295 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 8296 vsi->mqprio_qopt.max_rate[0]); 8297 8298 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 8299 if (!ret) { 8300 u64 credits = max_tx_rate; 8301 8302 do_div(credits, I40E_BW_CREDIT_DIVISOR); 8303 dev_dbg(&vsi->back->pdev->dev, 8304 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 8305 max_tx_rate, 8306 credits, 8307 vsi->seid); 8308 } else { 8309 need_reset = true; 8310 goto exit; 8311 } 8312 } 8313 ret = i40e_configure_queue_channels(vsi); 8314 if (ret) { 8315 vsi->num_queue_pairs = old_queue_pairs; 8316 netdev_info(netdev, 8317 "Failed configuring queue channels\n"); 8318 need_reset = true; 8319 goto exit; 8320 } 8321 } 8322 8323exit: 8324 /* Reset the configuration data to defaults, only TC0 is enabled */ 8325 if (need_reset) { 8326 i40e_vsi_set_default_tc_config(vsi); 8327 need_reset = false; 8328 } 8329 8330 /* Unquiesce VSI */ 8331 i40e_unquiesce_vsi(vsi); 8332 return ret; 8333} 8334 8335/** 8336 * i40e_set_cld_element - sets cloud filter element data 8337 * @filter: cloud filter rule 8338 * @cld: ptr to cloud filter element data 8339 * 8340 * This is helper function to copy data into cloud filter element 8341 **/ 8342static inline void 8343i40e_set_cld_element(struct i40e_cloud_filter *filter, 8344 struct i40e_aqc_cloud_filters_element_data *cld) 8345{ 8346 u32 ipa; 8347 int i; 8348 8349 memset(cld, 0, sizeof(*cld)); 8350 ether_addr_copy(cld->outer_mac, filter->dst_mac); 8351 ether_addr_copy(cld->inner_mac, filter->src_mac); 8352 8353 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 8354 return; 8355 8356 if (filter->n_proto == ETH_P_IPV6) { 8357#define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 8358 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) { 8359 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 8360 8361 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa); 8362 } 8363 } else { 8364 ipa = be32_to_cpu(filter->dst_ipv4); 8365 8366 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 8367 } 8368 8369 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 8370 8371 /* tenant_id is not supported by FW now, once the support is enabled 8372 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 8373 */ 8374 if (filter->tenant_id) 8375 return; 8376} 8377 8378/** 8379 * i40e_add_del_cloud_filter - Add/del cloud filter 8380 * @vsi: pointer to VSI 8381 * @filter: cloud filter rule 8382 * @add: if true, add, if false, delete 8383 * 8384 * Add or delete a cloud filter for a specific flow spec. 8385 * Returns 0 if the filter were successfully added. 8386 **/ 8387int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 8388 struct i40e_cloud_filter *filter, bool add) 8389{ 8390 struct i40e_aqc_cloud_filters_element_data cld_filter; 8391 struct i40e_pf *pf = vsi->back; 8392 int ret; 8393 static const u16 flag_table[128] = { 8394 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 8395 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 8396 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 8397 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 8398 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 8399 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 8400 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 8401 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 8402 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 8403 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 8404 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 8405 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 8406 [I40E_CLOUD_FILTER_FLAGS_IIP] = 8407 I40E_AQC_ADD_CLOUD_FILTER_IIP, 8408 }; 8409 8410 if (filter->flags >= ARRAY_SIZE(flag_table)) 8411 return -EIO; 8412 8413 memset(&cld_filter, 0, sizeof(cld_filter)); 8414 8415 /* copy element needed to add cloud filter from filter */ 8416 i40e_set_cld_element(filter, &cld_filter); 8417 8418 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 8419 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 8420 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 8421 8422 if (filter->n_proto == ETH_P_IPV6) 8423 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8424 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8425 else 8426 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8427 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8428 8429 if (add) 8430 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 8431 &cld_filter, 1); 8432 else 8433 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 8434 &cld_filter, 1); 8435 if (ret) 8436 dev_dbg(&pf->pdev->dev, 8437 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 8438 add ? "add" : "delete", filter->dst_port, ret, 8439 pf->hw.aq.asq_last_status); 8440 else 8441 dev_info(&pf->pdev->dev, 8442 "%s cloud filter for VSI: %d\n", 8443 add ? "Added" : "Deleted", filter->seid); 8444 return ret; 8445} 8446 8447/** 8448 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 8449 * @vsi: pointer to VSI 8450 * @filter: cloud filter rule 8451 * @add: if true, add, if false, delete 8452 * 8453 * Add or delete a cloud filter for a specific flow spec using big buffer. 8454 * Returns 0 if the filter were successfully added. 8455 **/ 8456int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 8457 struct i40e_cloud_filter *filter, 8458 bool add) 8459{ 8460 struct i40e_aqc_cloud_filters_element_bb cld_filter; 8461 struct i40e_pf *pf = vsi->back; 8462 int ret; 8463 8464 /* Both (src/dst) valid mac_addr are not supported */ 8465 if ((is_valid_ether_addr(filter->dst_mac) && 8466 is_valid_ether_addr(filter->src_mac)) || 8467 (is_multicast_ether_addr(filter->dst_mac) && 8468 is_multicast_ether_addr(filter->src_mac))) 8469 return -EOPNOTSUPP; 8470 8471 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 8472 * ports are not supported via big buffer now. 8473 */ 8474 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 8475 return -EOPNOTSUPP; 8476 8477 /* adding filter using src_port/src_ip is not supported at this stage */ 8478 if (filter->src_port || 8479 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) || 8480 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 8481 return -EOPNOTSUPP; 8482 8483 memset(&cld_filter, 0, sizeof(cld_filter)); 8484 8485 /* copy element needed to add cloud filter from filter */ 8486 i40e_set_cld_element(filter, &cld_filter.element); 8487 8488 if (is_valid_ether_addr(filter->dst_mac) || 8489 is_valid_ether_addr(filter->src_mac) || 8490 is_multicast_ether_addr(filter->dst_mac) || 8491 is_multicast_ether_addr(filter->src_mac)) { 8492 /* MAC + IP : unsupported mode */ 8493 if (filter->dst_ipv4) 8494 return -EOPNOTSUPP; 8495 8496 /* since we validated that L4 port must be valid before 8497 * we get here, start with respective "flags" value 8498 * and update if vlan is present or not 8499 */ 8500 cld_filter.element.flags = 8501 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 8502 8503 if (filter->vlan_id) { 8504 cld_filter.element.flags = 8505 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 8506 } 8507 8508 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) || 8509 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 8510 cld_filter.element.flags = 8511 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 8512 if (filter->n_proto == ETH_P_IPV6) 8513 cld_filter.element.flags |= 8514 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8515 else 8516 cld_filter.element.flags |= 8517 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8518 } else { 8519 dev_err(&pf->pdev->dev, 8520 "either mac or ip has to be valid for cloud filter\n"); 8521 return -EINVAL; 8522 } 8523 8524 /* Now copy L4 port in Byte 6..7 in general fields */ 8525 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 8526 be16_to_cpu(filter->dst_port); 8527 8528 if (add) { 8529 /* Validate current device switch mode, change if necessary */ 8530 ret = i40e_validate_and_set_switch_mode(vsi); 8531 if (ret) { 8532 dev_err(&pf->pdev->dev, 8533 "failed to set switch mode, ret %d\n", 8534 ret); 8535 return ret; 8536 } 8537 8538 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 8539 &cld_filter, 1); 8540 } else { 8541 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 8542 &cld_filter, 1); 8543 } 8544 8545 if (ret) 8546 dev_dbg(&pf->pdev->dev, 8547 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 8548 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 8549 else 8550 dev_info(&pf->pdev->dev, 8551 "%s cloud filter for VSI: %d, L4 port: %d\n", 8552 add ? "add" : "delete", filter->seid, 8553 ntohs(filter->dst_port)); 8554 return ret; 8555} 8556 8557/** 8558 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 8559 * @vsi: Pointer to VSI 8560 * @f: Pointer to struct flow_cls_offload 8561 * @filter: Pointer to cloud filter structure 8562 * 8563 **/ 8564static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 8565 struct flow_cls_offload *f, 8566 struct i40e_cloud_filter *filter) 8567{ 8568 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 8569 struct flow_dissector *dissector = rule->match.dissector; 8570 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 8571 struct i40e_pf *pf = vsi->back; 8572 u8 field_flags = 0; 8573 8574 if (dissector->used_keys & 8575 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 8576 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 8577 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 8578 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 8579 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 8580 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 8581 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 8582 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 8583 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n", 8584 dissector->used_keys); 8585 return -EOPNOTSUPP; 8586 } 8587 8588 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 8589 struct flow_match_enc_keyid match; 8590 8591 flow_rule_match_enc_keyid(rule, &match); 8592 if (match.mask->keyid != 0) 8593 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 8594 8595 filter->tenant_id = be32_to_cpu(match.key->keyid); 8596 } 8597 8598 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 8599 struct flow_match_basic match; 8600 8601 flow_rule_match_basic(rule, &match); 8602 n_proto_key = ntohs(match.key->n_proto); 8603 n_proto_mask = ntohs(match.mask->n_proto); 8604 8605 if (n_proto_key == ETH_P_ALL) { 8606 n_proto_key = 0; 8607 n_proto_mask = 0; 8608 } 8609 filter->n_proto = n_proto_key & n_proto_mask; 8610 filter->ip_proto = match.key->ip_proto; 8611 } 8612 8613 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 8614 struct flow_match_eth_addrs match; 8615 8616 flow_rule_match_eth_addrs(rule, &match); 8617 8618 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 8619 if (!is_zero_ether_addr(match.mask->dst)) { 8620 if (is_broadcast_ether_addr(match.mask->dst)) { 8621 field_flags |= I40E_CLOUD_FIELD_OMAC; 8622 } else { 8623 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 8624 match.mask->dst); 8625 return -EIO; 8626 } 8627 } 8628 8629 if (!is_zero_ether_addr(match.mask->src)) { 8630 if (is_broadcast_ether_addr(match.mask->src)) { 8631 field_flags |= I40E_CLOUD_FIELD_IMAC; 8632 } else { 8633 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 8634 match.mask->src); 8635 return -EIO; 8636 } 8637 } 8638 ether_addr_copy(filter->dst_mac, match.key->dst); 8639 ether_addr_copy(filter->src_mac, match.key->src); 8640 } 8641 8642 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 8643 struct flow_match_vlan match; 8644 8645 flow_rule_match_vlan(rule, &match); 8646 if (match.mask->vlan_id) { 8647 if (match.mask->vlan_id == VLAN_VID_MASK) { 8648 field_flags |= I40E_CLOUD_FIELD_IVLAN; 8649 8650 } else { 8651 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 8652 match.mask->vlan_id); 8653 return -EIO; 8654 } 8655 } 8656 8657 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 8658 } 8659 8660 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 8661 struct flow_match_control match; 8662 8663 flow_rule_match_control(rule, &match); 8664 addr_type = match.key->addr_type; 8665 8666 if (flow_rule_has_control_flags(match.mask->flags, 8667 f->common.extack)) 8668 return -EOPNOTSUPP; 8669 } 8670 8671 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 8672 struct flow_match_ipv4_addrs match; 8673 8674 flow_rule_match_ipv4_addrs(rule, &match); 8675 if (match.mask->dst) { 8676 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 8677 field_flags |= I40E_CLOUD_FIELD_IIP; 8678 } else { 8679 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 8680 &match.mask->dst); 8681 return -EIO; 8682 } 8683 } 8684 8685 if (match.mask->src) { 8686 if (match.mask->src == cpu_to_be32(0xffffffff)) { 8687 field_flags |= I40E_CLOUD_FIELD_IIP; 8688 } else { 8689 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 8690 &match.mask->src); 8691 return -EIO; 8692 } 8693 } 8694 8695 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 8696 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 8697 return -EIO; 8698 } 8699 filter->dst_ipv4 = match.key->dst; 8700 filter->src_ipv4 = match.key->src; 8701 } 8702 8703 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 8704 struct flow_match_ipv6_addrs match; 8705 8706 flow_rule_match_ipv6_addrs(rule, &match); 8707 8708 /* src and dest IPV6 address should not be LOOPBACK 8709 * (0:0:0:0:0:0:0:1), which can be represented as ::1 8710 */ 8711 if (ipv6_addr_loopback(&match.key->dst) || 8712 ipv6_addr_loopback(&match.key->src)) { 8713 dev_err(&pf->pdev->dev, 8714 "Bad ipv6, addr is LOOPBACK\n"); 8715 return -EIO; 8716 } 8717 if (!ipv6_addr_any(&match.mask->dst) || 8718 !ipv6_addr_any(&match.mask->src)) 8719 field_flags |= I40E_CLOUD_FIELD_IIP; 8720 8721 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 8722 sizeof(filter->src_ipv6)); 8723 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 8724 sizeof(filter->dst_ipv6)); 8725 } 8726 8727 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 8728 struct flow_match_ports match; 8729 8730 flow_rule_match_ports(rule, &match); 8731 if (match.mask->src) { 8732 if (match.mask->src == cpu_to_be16(0xffff)) { 8733 field_flags |= I40E_CLOUD_FIELD_IIP; 8734 } else { 8735 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 8736 be16_to_cpu(match.mask->src)); 8737 return -EIO; 8738 } 8739 } 8740 8741 if (match.mask->dst) { 8742 if (match.mask->dst == cpu_to_be16(0xffff)) { 8743 field_flags |= I40E_CLOUD_FIELD_IIP; 8744 } else { 8745 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 8746 be16_to_cpu(match.mask->dst)); 8747 return -EIO; 8748 } 8749 } 8750 8751 filter->dst_port = match.key->dst; 8752 filter->src_port = match.key->src; 8753 8754 switch (filter->ip_proto) { 8755 case IPPROTO_TCP: 8756 case IPPROTO_UDP: 8757 break; 8758 default: 8759 dev_err(&pf->pdev->dev, 8760 "Only UDP and TCP transport are supported\n"); 8761 return -EINVAL; 8762 } 8763 } 8764 filter->flags = field_flags; 8765 return 0; 8766} 8767 8768/** 8769 * i40e_handle_tclass: Forward to a traffic class on the device 8770 * @vsi: Pointer to VSI 8771 * @tc: traffic class index on the device 8772 * @filter: Pointer to cloud filter structure 8773 * 8774 **/ 8775static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 8776 struct i40e_cloud_filter *filter) 8777{ 8778 struct i40e_channel *ch, *ch_tmp; 8779 8780 /* direct to a traffic class on the same device */ 8781 if (tc == 0) { 8782 filter->seid = vsi->seid; 8783 return 0; 8784 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8785 if (!filter->dst_port) { 8786 dev_err(&vsi->back->pdev->dev, 8787 "Specify destination port to direct to traffic class that is not default\n"); 8788 return -EINVAL; 8789 } 8790 if (list_empty(&vsi->ch_list)) 8791 return -EINVAL; 8792 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8793 list) { 8794 if (ch->seid == vsi->tc_seid_map[tc]) 8795 filter->seid = ch->seid; 8796 } 8797 return 0; 8798 } 8799 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8800 return -EINVAL; 8801} 8802 8803/** 8804 * i40e_configure_clsflower - Configure tc flower filters 8805 * @vsi: Pointer to VSI 8806 * @cls_flower: Pointer to struct flow_cls_offload 8807 * 8808 **/ 8809static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8810 struct flow_cls_offload *cls_flower) 8811{ 8812 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8813 struct i40e_cloud_filter *filter = NULL; 8814 struct i40e_pf *pf = vsi->back; 8815 int err = 0; 8816 8817 if (tc < 0) { 8818 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8819 return -EOPNOTSUPP; 8820 } 8821 8822 if (!tc) { 8823 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination"); 8824 return -EINVAL; 8825 } 8826 8827 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8828 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8829 return -EBUSY; 8830 8831 if (pf->fdir_pf_active_filters || 8832 (!hlist_empty(&pf->fdir_filter_list))) { 8833 dev_err(&vsi->back->pdev->dev, 8834 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8835 return -EINVAL; 8836 } 8837 8838 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) { 8839 dev_err(&vsi->back->pdev->dev, 8840 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8841 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags); 8842 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags); 8843 } 8844 8845 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8846 if (!filter) 8847 return -ENOMEM; 8848 8849 filter->cookie = cls_flower->cookie; 8850 8851 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8852 if (err < 0) 8853 goto err; 8854 8855 err = i40e_handle_tclass(vsi, tc, filter); 8856 if (err < 0) 8857 goto err; 8858 8859 /* Add cloud filter */ 8860 if (filter->dst_port) 8861 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8862 else 8863 err = i40e_add_del_cloud_filter(vsi, filter, true); 8864 8865 if (err) { 8866 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n", 8867 err); 8868 goto err; 8869 } 8870 8871 /* add filter to the ordered list */ 8872 INIT_HLIST_NODE(&filter->cloud_node); 8873 8874 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8875 8876 pf->num_cloud_filters++; 8877 8878 return err; 8879err: 8880 kfree(filter); 8881 return err; 8882} 8883 8884/** 8885 * i40e_find_cloud_filter - Find the could filter in the list 8886 * @vsi: Pointer to VSI 8887 * @cookie: filter specific cookie 8888 * 8889 **/ 8890static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8891 unsigned long *cookie) 8892{ 8893 struct i40e_cloud_filter *filter = NULL; 8894 struct hlist_node *node2; 8895 8896 hlist_for_each_entry_safe(filter, node2, 8897 &vsi->back->cloud_filter_list, cloud_node) 8898 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8899 return filter; 8900 return NULL; 8901} 8902 8903/** 8904 * i40e_delete_clsflower - Remove tc flower filters 8905 * @vsi: Pointer to VSI 8906 * @cls_flower: Pointer to struct flow_cls_offload 8907 * 8908 **/ 8909static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8910 struct flow_cls_offload *cls_flower) 8911{ 8912 struct i40e_cloud_filter *filter = NULL; 8913 struct i40e_pf *pf = vsi->back; 8914 int err = 0; 8915 8916 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8917 8918 if (!filter) 8919 return -EINVAL; 8920 8921 hash_del(&filter->cloud_node); 8922 8923 if (filter->dst_port) 8924 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8925 else 8926 err = i40e_add_del_cloud_filter(vsi, filter, false); 8927 8928 kfree(filter); 8929 if (err) { 8930 dev_err(&pf->pdev->dev, 8931 "Failed to delete cloud filter, err %pe\n", 8932 ERR_PTR(err)); 8933 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8934 } 8935 8936 pf->num_cloud_filters--; 8937 if (!pf->num_cloud_filters) 8938 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 8939 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 8940 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 8941 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 8942 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 8943 } 8944 return 0; 8945} 8946 8947/** 8948 * i40e_setup_tc_cls_flower - flower classifier offloads 8949 * @np: net device to configure 8950 * @cls_flower: offload data 8951 **/ 8952static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8953 struct flow_cls_offload *cls_flower) 8954{ 8955 struct i40e_vsi *vsi = np->vsi; 8956 8957 switch (cls_flower->command) { 8958 case FLOW_CLS_REPLACE: 8959 return i40e_configure_clsflower(vsi, cls_flower); 8960 case FLOW_CLS_DESTROY: 8961 return i40e_delete_clsflower(vsi, cls_flower); 8962 case FLOW_CLS_STATS: 8963 return -EOPNOTSUPP; 8964 default: 8965 return -EOPNOTSUPP; 8966 } 8967} 8968 8969static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8970 void *cb_priv) 8971{ 8972 struct i40e_netdev_priv *np = cb_priv; 8973 8974 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8975 return -EOPNOTSUPP; 8976 8977 switch (type) { 8978 case TC_SETUP_CLSFLOWER: 8979 return i40e_setup_tc_cls_flower(np, type_data); 8980 8981 default: 8982 return -EOPNOTSUPP; 8983 } 8984} 8985 8986static LIST_HEAD(i40e_block_cb_list); 8987 8988static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8989 void *type_data) 8990{ 8991 struct i40e_netdev_priv *np = netdev_priv(netdev); 8992 8993 switch (type) { 8994 case TC_SETUP_QDISC_MQPRIO: 8995 return i40e_setup_tc(netdev, type_data); 8996 case TC_SETUP_BLOCK: 8997 return flow_block_cb_setup_simple(type_data, 8998 &i40e_block_cb_list, 8999 i40e_setup_tc_block_cb, 9000 np, np, true); 9001 default: 9002 return -EOPNOTSUPP; 9003 } 9004} 9005 9006/** 9007 * i40e_open - Called when a network interface is made active 9008 * @netdev: network interface device structure 9009 * 9010 * The open entry point is called when a network interface is made 9011 * active by the system (IFF_UP). At this point all resources needed 9012 * for transmit and receive operations are allocated, the interrupt 9013 * handler is registered with the OS, the netdev watchdog subtask is 9014 * enabled, and the stack is notified that the interface is ready. 9015 * 9016 * Returns 0 on success, negative value on failure 9017 **/ 9018int i40e_open(struct net_device *netdev) 9019{ 9020 struct i40e_netdev_priv *np = netdev_priv(netdev); 9021 struct i40e_vsi *vsi = np->vsi; 9022 struct i40e_pf *pf = vsi->back; 9023 int err; 9024 9025 /* disallow open during test or if eeprom is broken */ 9026 if (test_bit(__I40E_TESTING, pf->state) || 9027 test_bit(__I40E_BAD_EEPROM, pf->state)) 9028 return -EBUSY; 9029 9030 netif_carrier_off(netdev); 9031 9032 if (i40e_force_link_state(pf, true)) 9033 return -EAGAIN; 9034 9035 err = i40e_vsi_open(vsi); 9036 if (err) 9037 return err; 9038 9039 /* configure global TSO hardware offload settings */ 9040 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 9041 TCP_FLAG_FIN) >> 16); 9042 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 9043 TCP_FLAG_FIN | 9044 TCP_FLAG_CWR) >> 16); 9045 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 9046 9047 return 0; 9048} 9049 9050/** 9051 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues 9052 * @vsi: vsi structure 9053 * 9054 * This updates netdev's number of tx/rx queues 9055 * 9056 * Returns status of setting tx/rx queues 9057 **/ 9058static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi) 9059{ 9060 int ret; 9061 9062 ret = netif_set_real_num_rx_queues(vsi->netdev, 9063 vsi->num_queue_pairs); 9064 if (ret) 9065 return ret; 9066 9067 return netif_set_real_num_tx_queues(vsi->netdev, 9068 vsi->num_queue_pairs); 9069} 9070 9071/** 9072 * i40e_vsi_open - 9073 * @vsi: the VSI to open 9074 * 9075 * Finish initialization of the VSI. 9076 * 9077 * Returns 0 on success, negative value on failure 9078 * 9079 * Note: expects to be called while under rtnl_lock() 9080 **/ 9081int i40e_vsi_open(struct i40e_vsi *vsi) 9082{ 9083 struct i40e_pf *pf = vsi->back; 9084 char int_name[I40E_INT_NAME_STR_LEN]; 9085 int err; 9086 9087 /* allocate descriptors */ 9088 err = i40e_vsi_setup_tx_resources(vsi); 9089 if (err) 9090 goto err_setup_tx; 9091 err = i40e_vsi_setup_rx_resources(vsi); 9092 if (err) 9093 goto err_setup_rx; 9094 9095 err = i40e_vsi_configure(vsi); 9096 if (err) 9097 goto err_setup_rx; 9098 9099 if (vsi->netdev) { 9100 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 9101 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 9102 err = i40e_vsi_request_irq(vsi, int_name); 9103 if (err) 9104 goto err_setup_rx; 9105 9106 /* Notify the stack of the actual queue counts. */ 9107 err = i40e_netif_set_realnum_tx_rx_queues(vsi); 9108 if (err) 9109 goto err_set_queues; 9110 9111 } else if (vsi->type == I40E_VSI_FDIR) { 9112 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 9113 dev_driver_string(&pf->pdev->dev), 9114 dev_name(&pf->pdev->dev)); 9115 err = i40e_vsi_request_irq(vsi, int_name); 9116 if (err) 9117 goto err_setup_rx; 9118 9119 } else { 9120 err = -EINVAL; 9121 goto err_setup_rx; 9122 } 9123 9124 err = i40e_up_complete(vsi); 9125 if (err) 9126 goto err_up_complete; 9127 9128 return 0; 9129 9130err_up_complete: 9131 i40e_down(vsi); 9132err_set_queues: 9133 i40e_vsi_free_irq(vsi); 9134err_setup_rx: 9135 i40e_vsi_free_rx_resources(vsi); 9136err_setup_tx: 9137 i40e_vsi_free_tx_resources(vsi); 9138 if (vsi->type == I40E_VSI_MAIN) 9139 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 9140 9141 return err; 9142} 9143 9144/** 9145 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 9146 * @pf: Pointer to PF 9147 * 9148 * This function destroys the hlist where all the Flow Director 9149 * filters were saved. 9150 **/ 9151static void i40e_fdir_filter_exit(struct i40e_pf *pf) 9152{ 9153 struct i40e_fdir_filter *filter; 9154 struct i40e_flex_pit *pit_entry, *tmp; 9155 struct hlist_node *node2; 9156 9157 hlist_for_each_entry_safe(filter, node2, 9158 &pf->fdir_filter_list, fdir_node) { 9159 hlist_del(&filter->fdir_node); 9160 kfree(filter); 9161 } 9162 9163 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 9164 list_del(&pit_entry->list); 9165 kfree(pit_entry); 9166 } 9167 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 9168 9169 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 9170 list_del(&pit_entry->list); 9171 kfree(pit_entry); 9172 } 9173 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 9174 9175 pf->fdir_pf_active_filters = 0; 9176 i40e_reset_fdir_filter_cnt(pf); 9177 9178 /* Reprogram the default input set for TCP/IPv4 */ 9179 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_TCP, 9180 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9181 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9182 9183 /* Reprogram the default input set for TCP/IPv6 */ 9184 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_TCP, 9185 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9186 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9187 9188 /* Reprogram the default input set for UDP/IPv4 */ 9189 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_UDP, 9190 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9191 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9192 9193 /* Reprogram the default input set for UDP/IPv6 */ 9194 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_UDP, 9195 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9196 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9197 9198 /* Reprogram the default input set for SCTP/IPv4 */ 9199 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_SCTP, 9200 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9201 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9202 9203 /* Reprogram the default input set for SCTP/IPv6 */ 9204 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_SCTP, 9205 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9206 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9207 9208 /* Reprogram the default input set for Other/IPv4 */ 9209 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_OTHER, 9210 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9211 9212 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_FRAG_IPV4, 9213 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9214 9215 /* Reprogram the default input set for Other/IPv6 */ 9216 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV6_OTHER, 9217 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9218 9219 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_FRAG_IPV6, 9220 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9221} 9222 9223/** 9224 * i40e_cloud_filter_exit - Cleans up the cloud filters 9225 * @pf: Pointer to PF 9226 * 9227 * This function destroys the hlist where all the cloud filters 9228 * were saved. 9229 **/ 9230static void i40e_cloud_filter_exit(struct i40e_pf *pf) 9231{ 9232 struct i40e_cloud_filter *cfilter; 9233 struct hlist_node *node; 9234 9235 hlist_for_each_entry_safe(cfilter, node, 9236 &pf->cloud_filter_list, cloud_node) { 9237 hlist_del(&cfilter->cloud_node); 9238 kfree(cfilter); 9239 } 9240 pf->num_cloud_filters = 0; 9241 9242 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) && 9243 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) { 9244 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 9245 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags); 9246 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 9247 } 9248} 9249 9250/** 9251 * i40e_close - Disables a network interface 9252 * @netdev: network interface device structure 9253 * 9254 * The close entry point is called when an interface is de-activated 9255 * by the OS. The hardware is still under the driver's control, but 9256 * this netdev interface is disabled. 9257 * 9258 * Returns 0, this is not allowed to fail 9259 **/ 9260int i40e_close(struct net_device *netdev) 9261{ 9262 struct i40e_netdev_priv *np = netdev_priv(netdev); 9263 struct i40e_vsi *vsi = np->vsi; 9264 9265 i40e_vsi_close(vsi); 9266 9267 return 0; 9268} 9269 9270/** 9271 * i40e_do_reset - Start a PF or Core Reset sequence 9272 * @pf: board private structure 9273 * @reset_flags: which reset is requested 9274 * @lock_acquired: indicates whether or not the lock has been acquired 9275 * before this function was called. 9276 * 9277 * The essential difference in resets is that the PF Reset 9278 * doesn't clear the packet buffers, doesn't reset the PE 9279 * firmware, and doesn't bother the other PFs on the chip. 9280 **/ 9281void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 9282{ 9283 struct i40e_vsi *vsi; 9284 u32 val; 9285 int i; 9286 9287 /* do the biggest reset indicated */ 9288 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 9289 9290 /* Request a Global Reset 9291 * 9292 * This will start the chip's countdown to the actual full 9293 * chip reset event, and a warning interrupt to be sent 9294 * to all PFs, including the requestor. Our handler 9295 * for the warning interrupt will deal with the shutdown 9296 * and recovery of the switch setup. 9297 */ 9298 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 9299 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9300 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 9301 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9302 9303 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 9304 9305 /* Request a Core Reset 9306 * 9307 * Same as Global Reset, except does *not* include the MAC/PHY 9308 */ 9309 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 9310 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9311 val |= I40E_GLGEN_RTRIG_CORER_MASK; 9312 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9313 i40e_flush(&pf->hw); 9314 9315 } else if (reset_flags & I40E_PF_RESET_FLAG) { 9316 9317 /* Request a PF Reset 9318 * 9319 * Resets only the PF-specific registers 9320 * 9321 * This goes directly to the tear-down and rebuild of 9322 * the switch, since we need to do all the recovery as 9323 * for the Core Reset. 9324 */ 9325 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 9326 i40e_handle_reset_warning(pf, lock_acquired); 9327 9328 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) { 9329 /* Request a PF Reset 9330 * 9331 * Resets PF and reinitializes PFs VSI. 9332 */ 9333 i40e_prep_for_reset(pf); 9334 i40e_reset_and_rebuild(pf, true, lock_acquired); 9335 dev_info(&pf->pdev->dev, 9336 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 9337 "FW LLDP is disabled\n" : 9338 "FW LLDP is enabled\n"); 9339 9340 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 9341 /* Find the VSI(s) that requested a re-init */ 9342 dev_info(&pf->pdev->dev, "VSI reinit requested\n"); 9343 9344 i40e_pf_for_each_vsi(pf, i, vsi) { 9345 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 9346 vsi->state)) 9347 i40e_vsi_reinit_locked(vsi); 9348 } 9349 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 9350 /* Find the VSI(s) that needs to be brought down */ 9351 dev_info(&pf->pdev->dev, "VSI down requested\n"); 9352 9353 i40e_pf_for_each_vsi(pf, i, vsi) { 9354 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 9355 vsi->state)) { 9356 set_bit(__I40E_VSI_DOWN, vsi->state); 9357 i40e_down(vsi); 9358 } 9359 } 9360 } else { 9361 dev_info(&pf->pdev->dev, 9362 "bad reset request 0x%08x\n", reset_flags); 9363 } 9364} 9365 9366#ifdef CONFIG_I40E_DCB 9367/** 9368 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 9369 * @pf: board private structure 9370 * @old_cfg: current DCB config 9371 * @new_cfg: new DCB config 9372 **/ 9373bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 9374 struct i40e_dcbx_config *old_cfg, 9375 struct i40e_dcbx_config *new_cfg) 9376{ 9377 bool need_reconfig = false; 9378 9379 /* Check if ETS configuration has changed */ 9380 if (memcmp(&new_cfg->etscfg, 9381 &old_cfg->etscfg, 9382 sizeof(new_cfg->etscfg))) { 9383 /* If Priority Table has changed reconfig is needed */ 9384 if (memcmp(&new_cfg->etscfg.prioritytable, 9385 &old_cfg->etscfg.prioritytable, 9386 sizeof(new_cfg->etscfg.prioritytable))) { 9387 need_reconfig = true; 9388 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 9389 } 9390 9391 if (memcmp(&new_cfg->etscfg.tcbwtable, 9392 &old_cfg->etscfg.tcbwtable, 9393 sizeof(new_cfg->etscfg.tcbwtable))) 9394 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 9395 9396 if (memcmp(&new_cfg->etscfg.tsatable, 9397 &old_cfg->etscfg.tsatable, 9398 sizeof(new_cfg->etscfg.tsatable))) 9399 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 9400 } 9401 9402 /* Check if PFC configuration has changed */ 9403 if (memcmp(&new_cfg->pfc, 9404 &old_cfg->pfc, 9405 sizeof(new_cfg->pfc))) { 9406 need_reconfig = true; 9407 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 9408 } 9409 9410 /* Check if APP Table has changed */ 9411 if (memcmp(&new_cfg->app, 9412 &old_cfg->app, 9413 sizeof(new_cfg->app))) { 9414 need_reconfig = true; 9415 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 9416 } 9417 9418 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 9419 return need_reconfig; 9420} 9421 9422/** 9423 * i40e_handle_lldp_event - Handle LLDP Change MIB event 9424 * @pf: board private structure 9425 * @e: event info posted on ARQ 9426 **/ 9427static int i40e_handle_lldp_event(struct i40e_pf *pf, 9428 struct i40e_arq_event_info *e) 9429{ 9430 struct i40e_aqc_lldp_get_mib *mib = libie_aq_raw(&e->desc); 9431 struct i40e_hw *hw = &pf->hw; 9432 struct i40e_dcbx_config tmp_dcbx_cfg; 9433 bool need_reconfig = false; 9434 int ret = 0; 9435 u8 type; 9436 9437 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9438 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9439 (hw->phy.link_info.link_speed & 9440 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) && 9441 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9442 /* let firmware decide if the DCB should be disabled */ 9443 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9444 9445 /* Not DCB capable or capability disabled */ 9446 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 9447 return ret; 9448 9449 /* Ignore if event is not for Nearest Bridge */ 9450 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 9451 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 9452 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 9453 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 9454 return ret; 9455 9456 /* Check MIB Type and return if event for Remote MIB update */ 9457 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 9458 dev_dbg(&pf->pdev->dev, 9459 "LLDP event mib type %s\n", type ? "remote" : "local"); 9460 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 9461 /* Update the remote cached instance and return */ 9462 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 9463 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 9464 &hw->remote_dcbx_config); 9465 goto exit; 9466 } 9467 9468 /* Store the old configuration */ 9469 tmp_dcbx_cfg = hw->local_dcbx_config; 9470 9471 /* Reset the old DCBx configuration data */ 9472 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 9473 /* Get updated DCBX data from firmware */ 9474 ret = i40e_get_dcb_config(&pf->hw); 9475 if (ret) { 9476 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9477 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9478 (hw->phy.link_info.link_speed & 9479 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 9480 dev_warn(&pf->pdev->dev, 9481 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 9482 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9483 } else { 9484 dev_info(&pf->pdev->dev, 9485 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n", 9486 ERR_PTR(ret), 9487 libie_aq_str(pf->hw.aq.asq_last_status)); 9488 } 9489 goto exit; 9490 } 9491 9492 /* No change detected in DCBX configs */ 9493 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 9494 sizeof(tmp_dcbx_cfg))) { 9495 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 9496 goto exit; 9497 } 9498 9499 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 9500 &hw->local_dcbx_config); 9501 9502 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 9503 9504 if (!need_reconfig) 9505 goto exit; 9506 9507 /* Enable DCB tagging only when more than one TC */ 9508 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 9509 set_bit(I40E_FLAG_DCB_ENA, pf->flags); 9510 else 9511 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9512 9513 set_bit(__I40E_PORT_SUSPENDED, pf->state); 9514 /* Reconfiguration needed quiesce all VSIs */ 9515 i40e_pf_quiesce_all_vsi(pf); 9516 9517 /* Changes in configuration update VEB/VSI */ 9518 i40e_dcb_reconfigure(pf); 9519 9520 ret = i40e_resume_port_tx(pf); 9521 9522 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 9523 /* In case of error no point in resuming VSIs */ 9524 if (ret) 9525 goto exit; 9526 9527 /* Wait for the PF's queues to be disabled */ 9528 ret = i40e_pf_wait_queues_disabled(pf); 9529 if (ret) { 9530 /* Schedule PF reset to recover */ 9531 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9532 i40e_service_event_schedule(pf); 9533 } else { 9534 i40e_pf_unquiesce_all_vsi(pf); 9535 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 9536 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 9537 } 9538 9539exit: 9540 return ret; 9541} 9542#endif /* CONFIG_I40E_DCB */ 9543 9544/** 9545 * i40e_do_reset_safe - Protected reset path for userland calls. 9546 * @pf: board private structure 9547 * @reset_flags: which reset is requested 9548 * 9549 **/ 9550void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 9551{ 9552 rtnl_lock(); 9553 i40e_do_reset(pf, reset_flags, true); 9554 rtnl_unlock(); 9555} 9556 9557/** 9558 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 9559 * @pf: board private structure 9560 * @e: event info posted on ARQ 9561 * 9562 * Handler for LAN Queue Overflow Event generated by the firmware for PF 9563 * and VF queues 9564 **/ 9565static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 9566 struct i40e_arq_event_info *e) 9567{ 9568 struct i40e_aqc_lan_overflow *data = libie_aq_raw(&e->desc); 9569 u32 queue = le32_to_cpu(data->prtdcb_rupto); 9570 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 9571 struct i40e_hw *hw = &pf->hw; 9572 struct i40e_vf *vf; 9573 u16 vf_id; 9574 9575 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 9576 queue, qtx_ctl); 9577 9578 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) != 9579 I40E_QTX_CTL_VF_QUEUE) 9580 return; 9581 9582 /* Queue belongs to VF, find the VF and issue VF reset */ 9583 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl); 9584 vf_id -= hw->func_caps.vf_base_id; 9585 vf = &pf->vf[vf_id]; 9586 i40e_vc_notify_vf_reset(vf); 9587 /* Allow VF to process pending reset notification */ 9588 msleep(20); 9589 i40e_reset_vf(vf, false); 9590} 9591 9592/** 9593 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 9594 * @pf: board private structure 9595 **/ 9596u32 i40e_get_current_fd_count(struct i40e_pf *pf) 9597{ 9598 u32 val, fcnt_prog; 9599 9600 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9601 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 9602 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val); 9603 return fcnt_prog; 9604} 9605 9606/** 9607 * i40e_get_global_fd_count - Get total FD filters programmed on device 9608 * @pf: board private structure 9609 **/ 9610u32 i40e_get_global_fd_count(struct i40e_pf *pf) 9611{ 9612 u32 val, fcnt_prog; 9613 9614 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 9615 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 9616 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val); 9617 return fcnt_prog; 9618} 9619 9620/** 9621 * i40e_reenable_fdir_sb - Restore FDir SB capability 9622 * @pf: board private structure 9623 **/ 9624static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 9625{ 9626 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 9627 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 9628 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9629 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 9630} 9631 9632/** 9633 * i40e_reenable_fdir_atr - Restore FDir ATR capability 9634 * @pf: board private structure 9635 **/ 9636static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 9637{ 9638 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 9639 /* ATR uses the same filtering logic as SB rules. It only 9640 * functions properly if the input set mask is at the default 9641 * settings. It is safe to restore the default input set 9642 * because there are no active TCPv4 filter rules. 9643 */ 9644 i40e_write_fd_input_set(pf, LIBIE_FILTER_PCTYPE_NONF_IPV4_TCP, 9645 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9646 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9647 9648 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 9649 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9650 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 9651 } 9652} 9653 9654/** 9655 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 9656 * @pf: board private structure 9657 * @filter: FDir filter to remove 9658 */ 9659static void i40e_delete_invalid_filter(struct i40e_pf *pf, 9660 struct i40e_fdir_filter *filter) 9661{ 9662 /* Update counters */ 9663 pf->fdir_pf_active_filters--; 9664 pf->fd_inv = 0; 9665 9666 switch (filter->flow_type) { 9667 case TCP_V4_FLOW: 9668 pf->fd_tcp4_filter_cnt--; 9669 break; 9670 case UDP_V4_FLOW: 9671 pf->fd_udp4_filter_cnt--; 9672 break; 9673 case SCTP_V4_FLOW: 9674 pf->fd_sctp4_filter_cnt--; 9675 break; 9676 case TCP_V6_FLOW: 9677 pf->fd_tcp6_filter_cnt--; 9678 break; 9679 case UDP_V6_FLOW: 9680 pf->fd_udp6_filter_cnt--; 9681 break; 9682 case SCTP_V6_FLOW: 9683 pf->fd_udp6_filter_cnt--; 9684 break; 9685 case IP_USER_FLOW: 9686 switch (filter->ipl4_proto) { 9687 case IPPROTO_TCP: 9688 pf->fd_tcp4_filter_cnt--; 9689 break; 9690 case IPPROTO_UDP: 9691 pf->fd_udp4_filter_cnt--; 9692 break; 9693 case IPPROTO_SCTP: 9694 pf->fd_sctp4_filter_cnt--; 9695 break; 9696 case IPPROTO_IP: 9697 pf->fd_ip4_filter_cnt--; 9698 break; 9699 } 9700 break; 9701 case IPV6_USER_FLOW: 9702 switch (filter->ipl4_proto) { 9703 case IPPROTO_TCP: 9704 pf->fd_tcp6_filter_cnt--; 9705 break; 9706 case IPPROTO_UDP: 9707 pf->fd_udp6_filter_cnt--; 9708 break; 9709 case IPPROTO_SCTP: 9710 pf->fd_sctp6_filter_cnt--; 9711 break; 9712 case IPPROTO_IP: 9713 pf->fd_ip6_filter_cnt--; 9714 break; 9715 } 9716 break; 9717 } 9718 9719 /* Remove the filter from the list and free memory */ 9720 hlist_del(&filter->fdir_node); 9721 kfree(filter); 9722} 9723 9724/** 9725 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 9726 * @pf: board private structure 9727 **/ 9728void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 9729{ 9730 struct i40e_fdir_filter *filter; 9731 u32 fcnt_prog, fcnt_avail; 9732 struct hlist_node *node; 9733 9734 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9735 return; 9736 9737 /* Check if we have enough room to re-enable FDir SB capability. */ 9738 fcnt_prog = i40e_get_global_fd_count(pf); 9739 fcnt_avail = pf->fdir_pf_filter_count; 9740 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 9741 (pf->fd_add_err == 0) || 9742 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 9743 i40e_reenable_fdir_sb(pf); 9744 9745 /* We should wait for even more space before re-enabling ATR. 9746 * Additionally, we cannot enable ATR as long as we still have TCP SB 9747 * rules active. 9748 */ 9749 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 9750 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0) 9751 i40e_reenable_fdir_atr(pf); 9752 9753 /* if hw had a problem adding a filter, delete it */ 9754 if (pf->fd_inv > 0) { 9755 hlist_for_each_entry_safe(filter, node, 9756 &pf->fdir_filter_list, fdir_node) 9757 if (filter->fd_id == pf->fd_inv) 9758 i40e_delete_invalid_filter(pf, filter); 9759 } 9760} 9761 9762#define I40E_MIN_FD_FLUSH_INTERVAL 10 9763#define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 9764/** 9765 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 9766 * @pf: board private structure 9767 **/ 9768static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 9769{ 9770 unsigned long min_flush_time; 9771 int flush_wait_retry = 50; 9772 bool disable_atr = false; 9773 int fd_room; 9774 int reg; 9775 9776 if (!time_after(jiffies, pf->fd_flush_timestamp + 9777 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 9778 return; 9779 9780 /* If the flush is happening too quick and we have mostly SB rules we 9781 * should not re-enable ATR for some time. 9782 */ 9783 min_flush_time = pf->fd_flush_timestamp + 9784 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 9785 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 9786 9787 if (!(time_after(jiffies, min_flush_time)) && 9788 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 9789 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9790 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 9791 disable_atr = true; 9792 } 9793 9794 pf->fd_flush_timestamp = jiffies; 9795 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9796 /* flush all filters */ 9797 wr32(&pf->hw, I40E_PFQF_CTL_1, 9798 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 9799 i40e_flush(&pf->hw); 9800 pf->fd_flush_cnt++; 9801 pf->fd_add_err = 0; 9802 do { 9803 /* Check FD flush status every 5-6msec */ 9804 usleep_range(5000, 6000); 9805 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 9806 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 9807 break; 9808 } while (flush_wait_retry--); 9809 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 9810 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 9811 } else { 9812 /* replay sideband filters */ 9813 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf)); 9814 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 9815 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9816 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 9817 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9818 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 9819 } 9820} 9821 9822/** 9823 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed 9824 * @pf: board private structure 9825 **/ 9826u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 9827{ 9828 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 9829} 9830 9831/** 9832 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 9833 * @pf: board private structure 9834 **/ 9835static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 9836{ 9837 9838 /* if interface is down do nothing */ 9839 if (test_bit(__I40E_DOWN, pf->state)) 9840 return; 9841 9842 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9843 i40e_fdir_flush_and_replay(pf); 9844 9845 i40e_fdir_check_and_reenable(pf); 9846 9847} 9848 9849/** 9850 * i40e_vsi_link_event - notify VSI of a link event 9851 * @vsi: vsi to be notified 9852 * @link_up: link up or down 9853 **/ 9854static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9855{ 9856 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9857 return; 9858 9859 switch (vsi->type) { 9860 case I40E_VSI_MAIN: 9861 if (!vsi->netdev || !vsi->netdev_registered) 9862 break; 9863 9864 if (link_up) { 9865 netif_carrier_on(vsi->netdev); 9866 netif_tx_wake_all_queues(vsi->netdev); 9867 } else { 9868 netif_carrier_off(vsi->netdev); 9869 netif_tx_stop_all_queues(vsi->netdev); 9870 } 9871 break; 9872 9873 case I40E_VSI_SRIOV: 9874 case I40E_VSI_VMDQ2: 9875 case I40E_VSI_CTRL: 9876 case I40E_VSI_IWARP: 9877 case I40E_VSI_MIRROR: 9878 default: 9879 /* there is no notification for other VSIs */ 9880 break; 9881 } 9882} 9883 9884/** 9885 * i40e_veb_link_event - notify elements on the veb of a link event 9886 * @veb: veb to be notified 9887 * @link_up: link up or down 9888 **/ 9889static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9890{ 9891 struct i40e_vsi *vsi; 9892 struct i40e_pf *pf; 9893 int i; 9894 9895 if (!veb || !veb->pf) 9896 return; 9897 pf = veb->pf; 9898 9899 /* Send link event to contained VSIs */ 9900 i40e_pf_for_each_vsi(pf, i, vsi) 9901 if (vsi->uplink_seid == veb->seid) 9902 i40e_vsi_link_event(vsi, link_up); 9903} 9904 9905/** 9906 * i40e_link_event - Update netif_carrier status 9907 * @pf: board private structure 9908 **/ 9909static void i40e_link_event(struct i40e_pf *pf) 9910{ 9911 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 9912 struct i40e_veb *veb = i40e_pf_get_main_veb(pf); 9913 u8 new_link_speed, old_link_speed; 9914 bool new_link, old_link; 9915 int status; 9916#ifdef CONFIG_I40E_DCB 9917 int err; 9918#endif /* CONFIG_I40E_DCB */ 9919 9920 /* set this to force the get_link_status call to refresh state */ 9921 pf->hw.phy.get_link_info = true; 9922 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9923 status = i40e_get_link_status(&pf->hw, &new_link); 9924 9925 /* On success, disable temp link polling */ 9926 if (status == 0) { 9927 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9928 } else { 9929 /* Enable link polling temporarily until i40e_get_link_status 9930 * returns 0 9931 */ 9932 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9933 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9934 status); 9935 return; 9936 } 9937 9938 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9939 new_link_speed = pf->hw.phy.link_info.link_speed; 9940 9941 if (new_link == old_link && 9942 new_link_speed == old_link_speed && 9943 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9944 new_link == netif_carrier_ok(vsi->netdev))) 9945 return; 9946 9947 if (!new_link && old_link) 9948 pf->link_down_events++; 9949 9950 i40e_print_link_message(vsi, new_link); 9951 9952 /* Notify the base of the switch tree connected to 9953 * the link. Floating VEBs are not notified. 9954 */ 9955 if (veb) 9956 i40e_veb_link_event(veb, new_link); 9957 else 9958 i40e_vsi_link_event(vsi, new_link); 9959 9960 if (pf->vf) 9961 i40e_vc_notify_link_state(pf); 9962 9963 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 9964 i40e_ptp_set_increment(pf); 9965#ifdef CONFIG_I40E_DCB 9966 if (new_link == old_link) 9967 return; 9968 /* Not SW DCB so firmware will take care of default settings */ 9969 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED) 9970 return; 9971 9972 /* We cover here only link down, as after link up in case of SW DCB 9973 * SW LLDP agent will take care of setting it up 9974 */ 9975 if (!new_link) { 9976 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n"); 9977 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg)); 9978 err = i40e_dcb_sw_default_config(pf); 9979 if (err) { 9980 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9981 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9982 } else { 9983 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 9984 DCB_CAP_DCBX_VER_IEEE; 9985 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 9986 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 9987 } 9988 } 9989#endif /* CONFIG_I40E_DCB */ 9990} 9991 9992/** 9993 * i40e_watchdog_subtask - periodic checks not using event driven response 9994 * @pf: board private structure 9995 **/ 9996static void i40e_watchdog_subtask(struct i40e_pf *pf) 9997{ 9998 struct i40e_vsi *vsi; 9999 struct i40e_veb *veb; 10000 int i; 10001 10002 /* if interface is down do nothing */ 10003 if (test_bit(__I40E_DOWN, pf->state) || 10004 test_bit(__I40E_CONFIG_BUSY, pf->state)) 10005 return; 10006 10007 /* make sure we don't do these things too often */ 10008 if (time_before(jiffies, (pf->service_timer_previous + 10009 pf->service_timer_period))) 10010 return; 10011 pf->service_timer_previous = jiffies; 10012 10013 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) || 10014 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 10015 i40e_link_event(pf); 10016 10017 /* Update the stats for active netdevs so the network stack 10018 * can look at updated numbers whenever it cares to 10019 */ 10020 i40e_pf_for_each_vsi(pf, i, vsi) 10021 if (vsi->netdev) 10022 i40e_update_stats(vsi); 10023 10024 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) { 10025 /* Update the stats for the active switching components */ 10026 i40e_pf_for_each_veb(pf, i, veb) 10027 i40e_update_veb_stats(veb); 10028 } 10029 10030 i40e_ptp_rx_hang(pf); 10031 i40e_ptp_tx_hang(pf); 10032} 10033 10034/** 10035 * i40e_reset_subtask - Set up for resetting the device and driver 10036 * @pf: board private structure 10037 **/ 10038static void i40e_reset_subtask(struct i40e_pf *pf) 10039{ 10040 u32 reset_flags = 0; 10041 10042 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 10043 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 10044 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 10045 } 10046 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 10047 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 10048 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 10049 } 10050 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 10051 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 10052 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 10053 } 10054 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 10055 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 10056 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 10057 } 10058 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 10059 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 10060 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 10061 } 10062 10063 /* If there's a recovery already waiting, it takes 10064 * precedence before starting a new reset sequence. 10065 */ 10066 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 10067 i40e_prep_for_reset(pf); 10068 i40e_reset(pf); 10069 i40e_rebuild(pf, false, false); 10070 } 10071 10072 /* If we're already down or resetting, just bail */ 10073 if (reset_flags && 10074 !test_bit(__I40E_DOWN, pf->state) && 10075 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 10076 i40e_do_reset(pf, reset_flags, false); 10077 } 10078} 10079 10080/** 10081 * i40e_handle_link_event - Handle link event 10082 * @pf: board private structure 10083 * @e: event info posted on ARQ 10084 **/ 10085static void i40e_handle_link_event(struct i40e_pf *pf, 10086 struct i40e_arq_event_info *e) 10087{ 10088 struct i40e_aqc_get_link_status *status = libie_aq_raw(&e->desc); 10089 10090 /* Do a new status request to re-enable LSE reporting 10091 * and load new status information into the hw struct 10092 * This completely ignores any state information 10093 * in the ARQ event info, instead choosing to always 10094 * issue the AQ update link status command. 10095 */ 10096 i40e_link_event(pf); 10097 10098 /* Check if module meets thermal requirements */ 10099 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 10100 dev_err(&pf->pdev->dev, 10101 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 10102 dev_err(&pf->pdev->dev, 10103 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10104 } else { 10105 /* check for unqualified module, if link is down, suppress 10106 * the message if link was forced to be down. 10107 */ 10108 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 10109 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 10110 (!(status->link_info & I40E_AQ_LINK_UP)) && 10111 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) { 10112 dev_err(&pf->pdev->dev, 10113 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 10114 dev_err(&pf->pdev->dev, 10115 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10116 } 10117 } 10118} 10119 10120/** 10121 * i40e_clean_adminq_subtask - Clean the AdminQ rings 10122 * @pf: board private structure 10123 **/ 10124static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 10125{ 10126 struct i40e_arq_event_info event; 10127 struct i40e_hw *hw = &pf->hw; 10128 u16 pending, i = 0; 10129 u16 opcode; 10130 u32 oldval; 10131 int ret; 10132 u32 val; 10133 10134 /* Do not run clean AQ when PF reset fails */ 10135 if (test_bit(__I40E_RESET_FAILED, pf->state)) 10136 return; 10137 10138 /* check for error indications */ 10139 val = rd32(&pf->hw, I40E_PF_ARQLEN); 10140 oldval = val; 10141 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 10142 if (hw->debug_mask & I40E_DEBUG_AQ) 10143 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 10144 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 10145 } 10146 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 10147 if (hw->debug_mask & I40E_DEBUG_AQ) 10148 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 10149 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 10150 pf->arq_overflows++; 10151 } 10152 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 10153 if (hw->debug_mask & I40E_DEBUG_AQ) 10154 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 10155 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 10156 } 10157 if (oldval != val) 10158 wr32(&pf->hw, I40E_PF_ARQLEN, val); 10159 10160 val = rd32(&pf->hw, I40E_PF_ATQLEN); 10161 oldval = val; 10162 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 10163 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10164 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 10165 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 10166 } 10167 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 10168 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10169 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 10170 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 10171 } 10172 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 10173 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10174 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 10175 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 10176 } 10177 if (oldval != val) 10178 wr32(&pf->hw, I40E_PF_ATQLEN, val); 10179 10180 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 10181 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 10182 if (!event.msg_buf) 10183 return; 10184 10185 do { 10186 ret = i40e_clean_arq_element(hw, &event, &pending); 10187 if (ret == -EALREADY) 10188 break; 10189 else if (ret) { 10190 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 10191 break; 10192 } 10193 10194 opcode = le16_to_cpu(event.desc.opcode); 10195 switch (opcode) { 10196 10197 case i40e_aqc_opc_get_link_status: 10198 rtnl_lock(); 10199 i40e_handle_link_event(pf, &event); 10200 rtnl_unlock(); 10201 break; 10202 case i40e_aqc_opc_send_msg_to_pf: 10203 ret = i40e_vc_process_vf_msg(pf, 10204 le16_to_cpu(event.desc.retval), 10205 le32_to_cpu(event.desc.cookie_high), 10206 le32_to_cpu(event.desc.cookie_low), 10207 event.msg_buf, 10208 event.msg_len); 10209 break; 10210 case i40e_aqc_opc_lldp_update_mib: 10211 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 10212#ifdef CONFIG_I40E_DCB 10213 rtnl_lock(); 10214 i40e_handle_lldp_event(pf, &event); 10215 rtnl_unlock(); 10216#endif /* CONFIG_I40E_DCB */ 10217 break; 10218 case i40e_aqc_opc_event_lan_overflow: 10219 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 10220 i40e_handle_lan_overflow_event(pf, &event); 10221 break; 10222 case i40e_aqc_opc_send_msg_to_peer: 10223 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 10224 break; 10225 case i40e_aqc_opc_nvm_erase: 10226 case i40e_aqc_opc_nvm_update: 10227 case i40e_aqc_opc_oem_post_update: 10228 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 10229 "ARQ NVM operation 0x%04x completed\n", 10230 opcode); 10231 break; 10232 default: 10233 dev_info(&pf->pdev->dev, 10234 "ARQ: Unknown event 0x%04x ignored\n", 10235 opcode); 10236 break; 10237 } 10238 } while (i++ < I40E_AQ_WORK_LIMIT); 10239 10240 if (i < I40E_AQ_WORK_LIMIT) 10241 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 10242 10243 /* re-enable Admin queue interrupt cause */ 10244 val = rd32(hw, I40E_PFINT_ICR0_ENA); 10245 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 10246 wr32(hw, I40E_PFINT_ICR0_ENA, val); 10247 i40e_flush(hw); 10248 10249 kfree(event.msg_buf); 10250} 10251 10252/** 10253 * i40e_verify_eeprom - make sure eeprom is good to use 10254 * @pf: board private structure 10255 **/ 10256static void i40e_verify_eeprom(struct i40e_pf *pf) 10257{ 10258 int err; 10259 10260 err = i40e_diag_eeprom_test(&pf->hw); 10261 if (err) { 10262 /* retry in case of garbage read */ 10263 err = i40e_diag_eeprom_test(&pf->hw); 10264 if (err) { 10265 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 10266 err); 10267 set_bit(__I40E_BAD_EEPROM, pf->state); 10268 } 10269 } 10270 10271 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 10272 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 10273 clear_bit(__I40E_BAD_EEPROM, pf->state); 10274 } 10275} 10276 10277/** 10278 * i40e_enable_pf_switch_lb 10279 * @pf: pointer to the PF structure 10280 * 10281 * enable switch loop back or die - no point in a return value 10282 **/ 10283static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 10284{ 10285 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10286 struct i40e_vsi_context ctxt; 10287 int ret; 10288 10289 ctxt.seid = pf->main_vsi_seid; 10290 ctxt.pf_num = pf->hw.pf_id; 10291 ctxt.vf_num = 0; 10292 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10293 if (ret) { 10294 dev_info(&pf->pdev->dev, 10295 "couldn't get PF vsi config, err %pe aq_err %s\n", 10296 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10297 return; 10298 } 10299 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10300 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10301 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10302 10303 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10304 if (ret) { 10305 dev_info(&pf->pdev->dev, 10306 "update vsi switch failed, err %pe aq_err %s\n", 10307 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10308 } 10309} 10310 10311/** 10312 * i40e_disable_pf_switch_lb 10313 * @pf: pointer to the PF structure 10314 * 10315 * disable switch loop back or die - no point in a return value 10316 **/ 10317static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 10318{ 10319 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10320 struct i40e_vsi_context ctxt; 10321 int ret; 10322 10323 ctxt.seid = pf->main_vsi_seid; 10324 ctxt.pf_num = pf->hw.pf_id; 10325 ctxt.vf_num = 0; 10326 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10327 if (ret) { 10328 dev_info(&pf->pdev->dev, 10329 "couldn't get PF vsi config, err %pe aq_err %s\n", 10330 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10331 return; 10332 } 10333 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10334 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10335 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10336 10337 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10338 if (ret) { 10339 dev_info(&pf->pdev->dev, 10340 "update vsi switch failed, err %pe aq_err %s\n", 10341 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10342 } 10343} 10344 10345/** 10346 * i40e_config_bridge_mode - Configure the HW bridge mode 10347 * @veb: pointer to the bridge instance 10348 * 10349 * Configure the loop back mode for the LAN VSI that is downlink to the 10350 * specified HW bridge instance. It is expected this function is called 10351 * when a new HW bridge is instantiated. 10352 **/ 10353static void i40e_config_bridge_mode(struct i40e_veb *veb) 10354{ 10355 struct i40e_pf *pf = veb->pf; 10356 10357 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 10358 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 10359 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 10360 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 10361 i40e_disable_pf_switch_lb(pf); 10362 else 10363 i40e_enable_pf_switch_lb(pf); 10364} 10365 10366/** 10367 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it 10368 * @veb: pointer to the VEB instance 10369 * 10370 * This is a function that builds the attached VSIs. We track the connections 10371 * through our own index numbers because the seid's from the HW could change 10372 * across the reset. 10373 **/ 10374static int i40e_reconstitute_veb(struct i40e_veb *veb) 10375{ 10376 struct i40e_vsi *ctl_vsi = NULL; 10377 struct i40e_pf *pf = veb->pf; 10378 struct i40e_vsi *vsi; 10379 int v, ret; 10380 10381 /* As we do not maintain PV (port virtualizer) switch element then 10382 * there can be only one non-floating VEB that have uplink to MAC SEID 10383 * and its control VSI is the main one. 10384 */ 10385 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) { 10386 dev_err(&pf->pdev->dev, 10387 "Invalid uplink SEID for VEB %d\n", veb->idx); 10388 return -ENOENT; 10389 } 10390 10391 if (veb->uplink_seid == pf->mac_seid) { 10392 /* Check that the LAN VSI has VEB owning flag set */ 10393 ctl_vsi = i40e_pf_get_main_vsi(pf); 10394 10395 if (WARN_ON(ctl_vsi->veb_idx != veb->idx || 10396 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) { 10397 dev_err(&pf->pdev->dev, 10398 "Invalid control VSI for VEB %d\n", veb->idx); 10399 return -ENOENT; 10400 } 10401 10402 /* Add the control VSI to switch */ 10403 ret = i40e_add_vsi(ctl_vsi); 10404 if (ret) { 10405 dev_err(&pf->pdev->dev, 10406 "Rebuild of owner VSI for VEB %d failed: %d\n", 10407 veb->idx, ret); 10408 return ret; 10409 } 10410 10411 i40e_vsi_reset_stats(ctl_vsi); 10412 } 10413 10414 /* create the VEB in the switch and move the VSI onto the VEB */ 10415 ret = i40e_add_veb(veb, ctl_vsi); 10416 if (ret) 10417 return ret; 10418 10419 if (veb->uplink_seid) { 10420 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 10421 veb->bridge_mode = BRIDGE_MODE_VEB; 10422 else 10423 veb->bridge_mode = BRIDGE_MODE_VEPA; 10424 i40e_config_bridge_mode(veb); 10425 } 10426 10427 /* create the remaining VSIs attached to this VEB */ 10428 i40e_pf_for_each_vsi(pf, v, vsi) { 10429 if (vsi == ctl_vsi) 10430 continue; 10431 10432 if (vsi->veb_idx == veb->idx) { 10433 vsi->uplink_seid = veb->seid; 10434 ret = i40e_add_vsi(vsi); 10435 if (ret) { 10436 dev_info(&pf->pdev->dev, 10437 "rebuild of vsi_idx %d failed: %d\n", 10438 v, ret); 10439 return ret; 10440 } 10441 i40e_vsi_reset_stats(vsi); 10442 } 10443 } 10444 10445 return ret; 10446} 10447 10448/** 10449 * i40e_get_capabilities - get info about the HW 10450 * @pf: the PF struct 10451 * @list_type: AQ capability to be queried 10452 **/ 10453static int i40e_get_capabilities(struct i40e_pf *pf, 10454 enum i40e_admin_queue_opc list_type) 10455{ 10456 struct libie_aqc_list_caps_elem *cap_buf; 10457 u16 data_size; 10458 int buf_len; 10459 int err; 10460 10461 buf_len = 40 * sizeof(struct libie_aqc_list_caps_elem); 10462 do { 10463 cap_buf = kzalloc(buf_len, GFP_KERNEL); 10464 if (!cap_buf) 10465 return -ENOMEM; 10466 10467 /* this loads the data into the hw struct for us */ 10468 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 10469 &data_size, list_type, 10470 NULL); 10471 /* data loaded, buffer no longer needed */ 10472 kfree(cap_buf); 10473 10474 if (pf->hw.aq.asq_last_status == LIBIE_AQ_RC_ENOMEM) { 10475 /* retry with a larger buffer */ 10476 buf_len = data_size; 10477 } else if (pf->hw.aq.asq_last_status != LIBIE_AQ_RC_OK || err) { 10478 dev_info(&pf->pdev->dev, 10479 "capability discovery failed, err %pe aq_err %s\n", 10480 ERR_PTR(err), 10481 libie_aq_str(pf->hw.aq.asq_last_status)); 10482 return -ENODEV; 10483 } 10484 } while (err); 10485 10486 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 10487 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10488 dev_info(&pf->pdev->dev, 10489 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 10490 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 10491 pf->hw.func_caps.num_msix_vectors, 10492 pf->hw.func_caps.num_msix_vectors_vf, 10493 pf->hw.func_caps.fd_filters_guaranteed, 10494 pf->hw.func_caps.fd_filters_best_effort, 10495 pf->hw.func_caps.num_tx_qp, 10496 pf->hw.func_caps.num_vsis); 10497 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 10498 dev_info(&pf->pdev->dev, 10499 "switch_mode=0x%04x, function_valid=0x%08x\n", 10500 pf->hw.dev_caps.switch_mode, 10501 pf->hw.dev_caps.valid_functions); 10502 dev_info(&pf->pdev->dev, 10503 "SR-IOV=%d, num_vfs for all function=%u\n", 10504 pf->hw.dev_caps.sr_iov_1_1, 10505 pf->hw.dev_caps.num_vfs); 10506 dev_info(&pf->pdev->dev, 10507 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 10508 pf->hw.dev_caps.num_vsis, 10509 pf->hw.dev_caps.num_rx_qp, 10510 pf->hw.dev_caps.num_tx_qp); 10511 } 10512 } 10513 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10514#define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 10515 + pf->hw.func_caps.num_vfs) 10516 if (pf->hw.revision_id == 0 && 10517 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 10518 dev_info(&pf->pdev->dev, 10519 "got num_vsis %d, setting num_vsis to %d\n", 10520 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 10521 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 10522 } 10523 } 10524 return 0; 10525} 10526 10527static int i40e_vsi_clear(struct i40e_vsi *vsi); 10528 10529/** 10530 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 10531 * @pf: board private structure 10532 **/ 10533static void i40e_fdir_sb_setup(struct i40e_pf *pf) 10534{ 10535 struct i40e_vsi *main_vsi, *vsi; 10536 10537 /* quick workaround for an NVM issue that leaves a critical register 10538 * uninitialized 10539 */ 10540 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 10541 static const u32 hkey[] = { 10542 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 10543 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 10544 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 10545 0x95b3a76d}; 10546 int i; 10547 10548 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 10549 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 10550 } 10551 10552 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 10553 return; 10554 10555 /* find existing VSI and see if it needs configuring */ 10556 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10557 10558 /* create a new VSI if none exists */ 10559 if (!vsi) { 10560 main_vsi = i40e_pf_get_main_vsi(pf); 10561 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0); 10562 if (!vsi) { 10563 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 10564 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 10565 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 10566 return; 10567 } 10568 } 10569 10570 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 10571} 10572 10573/** 10574 * i40e_fdir_teardown - release the Flow Director resources 10575 * @pf: board private structure 10576 **/ 10577static void i40e_fdir_teardown(struct i40e_pf *pf) 10578{ 10579 struct i40e_vsi *vsi; 10580 10581 i40e_fdir_filter_exit(pf); 10582 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10583 if (vsi) 10584 i40e_vsi_release(vsi); 10585} 10586 10587/** 10588 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 10589 * @vsi: PF main vsi 10590 * @seid: seid of main or channel VSIs 10591 * 10592 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 10593 * existed before reset 10594 **/ 10595static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 10596{ 10597 struct i40e_cloud_filter *cfilter; 10598 struct i40e_pf *pf = vsi->back; 10599 struct hlist_node *node; 10600 int ret; 10601 10602 /* Add cloud filters back if they exist */ 10603 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 10604 cloud_node) { 10605 if (cfilter->seid != seid) 10606 continue; 10607 10608 if (cfilter->dst_port) 10609 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 10610 true); 10611 else 10612 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 10613 10614 if (ret) { 10615 dev_dbg(&pf->pdev->dev, 10616 "Failed to rebuild cloud filter, err %pe aq_err %s\n", 10617 ERR_PTR(ret), 10618 libie_aq_str(pf->hw.aq.asq_last_status)); 10619 return ret; 10620 } 10621 } 10622 return 0; 10623} 10624 10625/** 10626 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 10627 * @vsi: PF main vsi 10628 * 10629 * Rebuilds channel VSIs if they existed before reset 10630 **/ 10631static int i40e_rebuild_channels(struct i40e_vsi *vsi) 10632{ 10633 struct i40e_channel *ch, *ch_tmp; 10634 int ret; 10635 10636 if (list_empty(&vsi->ch_list)) 10637 return 0; 10638 10639 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 10640 if (!ch->initialized) 10641 break; 10642 /* Proceed with creation of channel (VMDq2) VSI */ 10643 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 10644 if (ret) { 10645 dev_info(&vsi->back->pdev->dev, 10646 "failed to rebuild channels using uplink_seid %u\n", 10647 vsi->uplink_seid); 10648 return ret; 10649 } 10650 /* Reconfigure TX queues using QTX_CTL register */ 10651 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 10652 if (ret) { 10653 dev_info(&vsi->back->pdev->dev, 10654 "failed to configure TX rings for channel %u\n", 10655 ch->seid); 10656 return ret; 10657 } 10658 /* update 'next_base_queue' */ 10659 vsi->next_base_queue = vsi->next_base_queue + 10660 ch->num_queue_pairs; 10661 if (ch->max_tx_rate) { 10662 u64 credits = ch->max_tx_rate; 10663 10664 if (i40e_set_bw_limit(vsi, ch->seid, 10665 ch->max_tx_rate)) 10666 return -EINVAL; 10667 10668 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10669 dev_dbg(&vsi->back->pdev->dev, 10670 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10671 ch->max_tx_rate, 10672 credits, 10673 ch->seid); 10674 } 10675 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 10676 if (ret) { 10677 dev_dbg(&vsi->back->pdev->dev, 10678 "Failed to rebuild cloud filters for channel VSI %u\n", 10679 ch->seid); 10680 return ret; 10681 } 10682 } 10683 return 0; 10684} 10685 10686/** 10687 * i40e_clean_xps_state - clean xps state for every tx_ring 10688 * @vsi: ptr to the VSI 10689 **/ 10690static void i40e_clean_xps_state(struct i40e_vsi *vsi) 10691{ 10692 int i; 10693 10694 if (vsi->tx_rings) 10695 for (i = 0; i < vsi->num_queue_pairs; i++) 10696 if (vsi->tx_rings[i]) 10697 clear_bit(__I40E_TX_XPS_INIT_DONE, 10698 vsi->tx_rings[i]->state); 10699} 10700 10701/** 10702 * i40e_prep_for_reset - prep for the core to reset 10703 * @pf: board private structure 10704 * 10705 * Close up the VFs and other things in prep for PF Reset. 10706 **/ 10707static void i40e_prep_for_reset(struct i40e_pf *pf) 10708{ 10709 struct i40e_hw *hw = &pf->hw; 10710 struct i40e_vsi *vsi; 10711 int ret = 0; 10712 u32 v; 10713 10714 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 10715 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 10716 return; 10717 if (i40e_check_asq_alive(&pf->hw)) 10718 i40e_vc_notify_reset(pf); 10719 10720 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 10721 10722 /* quiesce the VSIs and their queues that are not already DOWN */ 10723 i40e_pf_quiesce_all_vsi(pf); 10724 10725 i40e_pf_for_each_vsi(pf, v, vsi) { 10726 i40e_clean_xps_state(vsi); 10727 vsi->seid = 0; 10728 } 10729 10730 i40e_shutdown_adminq(&pf->hw); 10731 10732 /* call shutdown HMC */ 10733 if (hw->hmc.hmc_obj) { 10734 ret = i40e_shutdown_lan_hmc(hw); 10735 if (ret) 10736 dev_warn(&pf->pdev->dev, 10737 "shutdown_lan_hmc failed: %d\n", ret); 10738 } 10739 10740 /* Save the current PTP time so that we can restore the time after the 10741 * reset completes. 10742 */ 10743 i40e_ptp_save_hw_time(pf); 10744} 10745 10746/** 10747 * i40e_send_version - update firmware with driver version 10748 * @pf: PF struct 10749 */ 10750static void i40e_send_version(struct i40e_pf *pf) 10751{ 10752 struct i40e_driver_version dv; 10753 10754 dv.major_version = 0xff; 10755 dv.minor_version = 0xff; 10756 dv.build_version = 0xff; 10757 dv.subbuild_version = 0; 10758 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 10759 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 10760} 10761 10762/** 10763 * i40e_get_oem_version - get OEM specific version information 10764 * @hw: pointer to the hardware structure 10765 **/ 10766static void i40e_get_oem_version(struct i40e_hw *hw) 10767{ 10768 u16 block_offset = 0xffff; 10769 u16 block_length = 0; 10770 u16 capabilities = 0; 10771 u16 gen_snap = 0; 10772 u16 release = 0; 10773 10774#define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 10775#define I40E_NVM_OEM_LENGTH_OFFSET 0x00 10776#define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 10777#define I40E_NVM_OEM_GEN_OFFSET 0x02 10778#define I40E_NVM_OEM_RELEASE_OFFSET 0x03 10779#define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 10780#define I40E_NVM_OEM_LENGTH 3 10781 10782 /* Check if pointer to OEM version block is valid. */ 10783 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 10784 if (block_offset == 0xffff) 10785 return; 10786 10787 /* Check if OEM version block has correct length. */ 10788 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 10789 &block_length); 10790 if (block_length < I40E_NVM_OEM_LENGTH) 10791 return; 10792 10793 /* Check if OEM version format is as expected. */ 10794 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 10795 &capabilities); 10796 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 10797 return; 10798 10799 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 10800 &gen_snap); 10801 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 10802 &release); 10803 hw->nvm.oem_ver = 10804 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) | 10805 FIELD_PREP(I40E_OEM_RELEASE_MASK, release); 10806 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 10807} 10808 10809/** 10810 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 10811 * @pf: board private structure 10812 **/ 10813static int i40e_reset(struct i40e_pf *pf) 10814{ 10815 struct i40e_hw *hw = &pf->hw; 10816 int ret; 10817 10818 ret = i40e_pf_reset(hw); 10819 if (ret) { 10820 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 10821 set_bit(__I40E_RESET_FAILED, pf->state); 10822 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10823 } else { 10824 pf->pfr_count++; 10825 } 10826 return ret; 10827} 10828 10829/** 10830 * i40e_rebuild - rebuild using a saved config 10831 * @pf: board private structure 10832 * @reinit: if the Main VSI needs to re-initialized. 10833 * @lock_acquired: indicates whether or not the lock has been acquired 10834 * before this function was called. 10835 **/ 10836static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 10837{ 10838 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); 10839 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 10840 struct i40e_hw *hw = &pf->hw; 10841 struct i40e_veb *veb; 10842 int ret; 10843 u32 val; 10844 int v; 10845 10846 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10847 is_recovery_mode_reported) 10848 i40e_set_ethtool_ops(vsi->netdev); 10849 10850 if (test_bit(__I40E_DOWN, pf->state) && 10851 !test_bit(__I40E_RECOVERY_MODE, pf->state)) 10852 goto clear_recovery; 10853 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 10854 10855 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 10856 ret = i40e_init_adminq(&pf->hw); 10857 if (ret) { 10858 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n", 10859 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10860 goto clear_recovery; 10861 } 10862 i40e_get_oem_version(&pf->hw); 10863 10864 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) { 10865 /* The following delay is necessary for firmware update. */ 10866 mdelay(1000); 10867 } 10868 10869 /* re-verify the eeprom if we just had an EMP reset */ 10870 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10871 i40e_verify_eeprom(pf); 10872 10873 /* if we are going out of or into recovery mode we have to act 10874 * accordingly with regard to resources initialization 10875 * and deinitialization 10876 */ 10877 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10878 if (i40e_get_capabilities(pf, 10879 i40e_aqc_opc_list_func_capabilities)) 10880 goto end_unlock; 10881 10882 if (is_recovery_mode_reported) { 10883 /* we're staying in recovery mode so we'll reinitialize 10884 * misc vector here 10885 */ 10886 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10887 goto end_unlock; 10888 } else { 10889 if (!lock_acquired) 10890 rtnl_lock(); 10891 /* we're going out of recovery mode so we'll free 10892 * the IRQ allocated specifically for recovery mode 10893 * and restore the interrupt scheme 10894 */ 10895 free_irq(pf->pdev->irq, pf); 10896 i40e_clear_interrupt_scheme(pf); 10897 if (i40e_restore_interrupt_scheme(pf)) 10898 goto end_unlock; 10899 } 10900 10901 /* tell the firmware that we're starting */ 10902 i40e_send_version(pf); 10903 10904 /* bail out in case recovery mode was detected, as there is 10905 * no need for further configuration. 10906 */ 10907 goto end_unlock; 10908 } 10909 10910 i40e_clear_pxe_mode(hw); 10911 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10912 if (ret) 10913 goto end_core_reset; 10914 10915 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10916 hw->func_caps.num_rx_qp, 0, 0); 10917 if (ret) { 10918 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10919 goto end_core_reset; 10920 } 10921 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10922 if (ret) { 10923 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10924 goto end_core_reset; 10925 } 10926 10927#ifdef CONFIG_I40E_DCB 10928 /* Enable FW to write a default DCB config on link-up 10929 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB 10930 * is not supported with new link speed 10931 */ 10932 if (i40e_is_tc_mqprio_enabled(pf)) { 10933 i40e_aq_set_dcb_parameters(hw, false, NULL); 10934 } else { 10935 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 10936 (hw->phy.link_info.link_speed & 10937 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 10938 i40e_aq_set_dcb_parameters(hw, false, NULL); 10939 dev_warn(&pf->pdev->dev, 10940 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 10941 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10942 } else { 10943 i40e_aq_set_dcb_parameters(hw, true, NULL); 10944 ret = i40e_init_pf_dcb(pf); 10945 if (ret) { 10946 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", 10947 ret); 10948 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 10949 /* Continue without DCB enabled */ 10950 } 10951 } 10952 } 10953 10954#endif /* CONFIG_I40E_DCB */ 10955 if (!lock_acquired) 10956 rtnl_lock(); 10957 ret = i40e_setup_pf_switch(pf, reinit, true); 10958 if (ret) 10959 goto end_unlock; 10960 10961 /* The driver only wants link up/down and module qualification 10962 * reports from firmware. Note the negative logic. 10963 */ 10964 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10965 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10966 I40E_AQ_EVENT_MEDIA_NA | 10967 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10968 if (ret) 10969 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 10970 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 10971 10972 /* Rebuild the VSIs and VEBs that existed before reset. 10973 * They are still in our local switch element arrays, so only 10974 * need to rebuild the switch model in the HW. 10975 * 10976 * If there were VEBs but the reconstitution failed, we'll try 10977 * to recover minimal use by getting the basic PF VSI working. 10978 */ 10979 if (vsi->uplink_seid != pf->mac_seid) { 10980 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10981 10982 /* Rebuild VEBs */ 10983 i40e_pf_for_each_veb(pf, v, veb) { 10984 ret = i40e_reconstitute_veb(veb); 10985 if (!ret) 10986 continue; 10987 10988 /* If Main VEB failed, we're in deep doodoo, 10989 * so give up rebuilding the switch and set up 10990 * for minimal rebuild of PF VSI. 10991 * If orphan failed, we'll report the error 10992 * but try to keep going. 10993 */ 10994 if (veb->uplink_seid == pf->mac_seid) { 10995 dev_info(&pf->pdev->dev, 10996 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10997 ret); 10998 vsi->uplink_seid = pf->mac_seid; 10999 break; 11000 } else if (veb->uplink_seid == 0) { 11001 dev_info(&pf->pdev->dev, 11002 "rebuild of orphan VEB failed: %d\n", 11003 ret); 11004 } 11005 } 11006 } 11007 11008 if (vsi->uplink_seid == pf->mac_seid) { 11009 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 11010 /* no VEB, so rebuild only the Main VSI */ 11011 ret = i40e_add_vsi(vsi); 11012 if (ret) { 11013 dev_info(&pf->pdev->dev, 11014 "rebuild of Main VSI failed: %d\n", ret); 11015 goto end_unlock; 11016 } 11017 } 11018 11019 if (vsi->mqprio_qopt.max_rate[0]) { 11020 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 11021 vsi->mqprio_qopt.max_rate[0]); 11022 u64 credits = 0; 11023 11024 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 11025 if (ret) 11026 goto end_unlock; 11027 11028 credits = max_tx_rate; 11029 do_div(credits, I40E_BW_CREDIT_DIVISOR); 11030 dev_dbg(&vsi->back->pdev->dev, 11031 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 11032 max_tx_rate, 11033 credits, 11034 vsi->seid); 11035 } 11036 11037 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 11038 if (ret) 11039 goto end_unlock; 11040 11041 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 11042 * for this main VSI if they exist 11043 */ 11044 ret = i40e_rebuild_channels(vsi); 11045 if (ret) 11046 goto end_unlock; 11047 11048 /* Reconfigure hardware for allowing smaller MSS in the case 11049 * of TSO, so that we avoid the MDD being fired and causing 11050 * a reset in the case of small MSS+TSO. 11051 */ 11052#define I40E_REG_MSS 0x000E64DC 11053#define I40E_REG_MSS_MIN_MASK 0x3FF0000 11054#define I40E_64BYTE_MSS 0x400000 11055 val = rd32(hw, I40E_REG_MSS); 11056 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11057 val &= ~I40E_REG_MSS_MIN_MASK; 11058 val |= I40E_64BYTE_MSS; 11059 wr32(hw, I40E_REG_MSS, val); 11060 } 11061 11062 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 11063 msleep(75); 11064 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11065 if (ret) 11066 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 11067 ERR_PTR(ret), 11068 libie_aq_str(pf->hw.aq.asq_last_status)); 11069 } 11070 /* reinit the misc interrupt */ 11071 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 11072 ret = i40e_setup_misc_vector(pf); 11073 if (ret) 11074 goto end_unlock; 11075 } 11076 11077 /* Add a filter to drop all Flow control frames from any VSI from being 11078 * transmitted. By doing so we stop a malicious VF from sending out 11079 * PAUSE or PFC frames and potentially controlling traffic for other 11080 * PF/VF VSIs. 11081 * The FW can still send Flow control frames if enabled. 11082 */ 11083 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11084 pf->main_vsi_seid); 11085 11086 /* restart the VSIs that were rebuilt and running before the reset */ 11087 i40e_pf_unquiesce_all_vsi(pf); 11088 11089 /* Release the RTNL lock before we start resetting VFs */ 11090 if (!lock_acquired) 11091 rtnl_unlock(); 11092 11093 /* Restore promiscuous settings */ 11094 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 11095 if (ret) 11096 dev_warn(&pf->pdev->dev, 11097 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n", 11098 pf->cur_promisc ? "on" : "off", 11099 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 11100 11101 i40e_reset_all_vfs(pf, true); 11102 11103 /* tell the firmware that we're starting */ 11104 i40e_send_version(pf); 11105 11106 /* We've already released the lock, so don't do it again */ 11107 goto end_core_reset; 11108 11109end_unlock: 11110 if (!lock_acquired) 11111 rtnl_unlock(); 11112end_core_reset: 11113 clear_bit(__I40E_RESET_FAILED, pf->state); 11114clear_recovery: 11115 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 11116 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 11117} 11118 11119/** 11120 * i40e_reset_and_rebuild - reset and rebuild using a saved config 11121 * @pf: board private structure 11122 * @reinit: if the Main VSI needs to re-initialized. 11123 * @lock_acquired: indicates whether or not the lock has been acquired 11124 * before this function was called. 11125 **/ 11126static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 11127 bool lock_acquired) 11128{ 11129 int ret; 11130 11131 if (test_bit(__I40E_IN_REMOVE, pf->state)) 11132 return; 11133 /* Now we wait for GRST to settle out. 11134 * We don't have to delete the VEBs or VSIs from the hw switch 11135 * because the reset will make them disappear. 11136 */ 11137 ret = i40e_reset(pf); 11138 if (!ret) 11139 i40e_rebuild(pf, reinit, lock_acquired); 11140 else 11141 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__); 11142} 11143 11144/** 11145 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 11146 * @pf: board private structure 11147 * 11148 * Close up the VFs and other things in prep for a Core Reset, 11149 * then get ready to rebuild the world. 11150 * @lock_acquired: indicates whether or not the lock has been acquired 11151 * before this function was called. 11152 **/ 11153static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 11154{ 11155 i40e_prep_for_reset(pf); 11156 i40e_reset_and_rebuild(pf, false, lock_acquired); 11157} 11158 11159/** 11160 * i40e_print_vf_mdd_event - print VF Tx/Rx malicious driver detect event 11161 * @pf: board private structure 11162 * @vf: pointer to the VF structure 11163 * @is_tx: true - for Tx event, false - for Rx 11164 */ 11165static void i40e_print_vf_mdd_event(struct i40e_pf *pf, struct i40e_vf *vf, 11166 bool is_tx) 11167{ 11168 dev_err(&pf->pdev->dev, is_tx ? 11169 "%lld Tx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n" : 11170 "%lld Rx Malicious Driver Detection events detected on PF %d VF %d MAC %pm. mdd-auto-reset-vfs=%s\n", 11171 is_tx ? vf->mdd_tx_events.count : vf->mdd_rx_events.count, 11172 pf->hw.pf_id, 11173 vf->vf_id, 11174 vf->default_lan_addr.addr, 11175 str_on_off(test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags))); 11176} 11177 11178/** 11179 * i40e_print_vfs_mdd_events - print VFs malicious driver detect event 11180 * @pf: pointer to the PF structure 11181 * 11182 * Called from i40e_handle_mdd_event to rate limit and print VFs MDD events. 11183 */ 11184static void i40e_print_vfs_mdd_events(struct i40e_pf *pf) 11185{ 11186 unsigned int i; 11187 11188 /* check that there are pending MDD events to print */ 11189 if (!test_and_clear_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state)) 11190 return; 11191 11192 if (!__ratelimit(&pf->mdd_message_rate_limit)) 11193 return; 11194 11195 for (i = 0; i < pf->num_alloc_vfs; i++) { 11196 struct i40e_vf *vf = &pf->vf[i]; 11197 bool is_printed = false; 11198 11199 /* only print Rx MDD event message if there are new events */ 11200 if (vf->mdd_rx_events.count != vf->mdd_rx_events.last_printed) { 11201 vf->mdd_rx_events.last_printed = vf->mdd_rx_events.count; 11202 i40e_print_vf_mdd_event(pf, vf, false); 11203 is_printed = true; 11204 } 11205 11206 /* only print Tx MDD event message if there are new events */ 11207 if (vf->mdd_tx_events.count != vf->mdd_tx_events.last_printed) { 11208 vf->mdd_tx_events.last_printed = vf->mdd_tx_events.count; 11209 i40e_print_vf_mdd_event(pf, vf, true); 11210 is_printed = true; 11211 } 11212 11213 if (is_printed && !test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)) 11214 dev_info(&pf->pdev->dev, 11215 "Use PF Control I/F to re-enable the VF #%d\n", 11216 i); 11217 } 11218} 11219 11220/** 11221 * i40e_handle_mdd_event 11222 * @pf: pointer to the PF structure 11223 * 11224 * Called from the MDD irq handler to identify possibly malicious vfs 11225 **/ 11226static void i40e_handle_mdd_event(struct i40e_pf *pf) 11227{ 11228 struct i40e_hw *hw = &pf->hw; 11229 bool mdd_detected = false; 11230 struct i40e_vf *vf; 11231 u32 reg; 11232 int i; 11233 11234 if (!test_and_clear_bit(__I40E_MDD_EVENT_PENDING, pf->state)) { 11235 /* Since the VF MDD event logging is rate limited, check if 11236 * there are pending MDD events. 11237 */ 11238 i40e_print_vfs_mdd_events(pf); 11239 return; 11240 } 11241 11242 /* find what triggered the MDD event */ 11243 reg = rd32(hw, I40E_GL_MDET_TX); 11244 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 11245 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg); 11246 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg); 11247 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg); 11248 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) - 11249 pf->hw.func_caps.base_queue; 11250 if (netif_msg_tx_err(pf)) 11251 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 11252 event, queue, pf_num, vf_num); 11253 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 11254 mdd_detected = true; 11255 } 11256 reg = rd32(hw, I40E_GL_MDET_RX); 11257 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 11258 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg); 11259 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg); 11260 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) - 11261 pf->hw.func_caps.base_queue; 11262 if (netif_msg_rx_err(pf)) 11263 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 11264 event, queue, func); 11265 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 11266 mdd_detected = true; 11267 } 11268 11269 if (mdd_detected) { 11270 reg = rd32(hw, I40E_PF_MDET_TX); 11271 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 11272 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 11273 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 11274 } 11275 reg = rd32(hw, I40E_PF_MDET_RX); 11276 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 11277 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 11278 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 11279 } 11280 } 11281 11282 /* see if one of the VFs needs its hand slapped */ 11283 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 11284 bool is_mdd_on_tx = false; 11285 bool is_mdd_on_rx = false; 11286 11287 vf = &(pf->vf[i]); 11288 reg = rd32(hw, I40E_VP_MDET_TX(i)); 11289 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 11290 set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state); 11291 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 11292 vf->mdd_tx_events.count++; 11293 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11294 is_mdd_on_tx = true; 11295 } 11296 11297 reg = rd32(hw, I40E_VP_MDET_RX(i)); 11298 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 11299 set_bit(__I40E_MDD_VF_PRINT_PENDING, pf->state); 11300 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 11301 vf->mdd_rx_events.count++; 11302 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11303 is_mdd_on_rx = true; 11304 } 11305 11306 if ((is_mdd_on_tx || is_mdd_on_rx) && 11307 test_bit(I40E_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 11308 /* VF MDD event counters will be cleared by 11309 * reset, so print the event prior to reset. 11310 */ 11311 if (is_mdd_on_rx) 11312 i40e_print_vf_mdd_event(pf, vf, false); 11313 if (is_mdd_on_tx) 11314 i40e_print_vf_mdd_event(pf, vf, true); 11315 11316 i40e_vc_reset_vf(vf, true); 11317 } 11318 } 11319 11320 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 11321 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 11322 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 11323 i40e_flush(hw); 11324 11325 i40e_print_vfs_mdd_events(pf); 11326} 11327 11328/** 11329 * i40e_service_task - Run the driver's async subtasks 11330 * @work: pointer to work_struct containing our data 11331 **/ 11332static void i40e_service_task(struct work_struct *work) 11333{ 11334 struct i40e_pf *pf = container_of(work, 11335 struct i40e_pf, 11336 service_task); 11337 unsigned long start_time = jiffies; 11338 11339 /* don't bother with service tasks if a reset is in progress */ 11340 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 11341 test_bit(__I40E_SUSPENDED, pf->state)) 11342 return; 11343 11344 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 11345 return; 11346 11347 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 11348 i40e_detect_recover_hung(pf); 11349 i40e_sync_filters_subtask(pf); 11350 i40e_reset_subtask(pf); 11351 i40e_handle_mdd_event(pf); 11352 i40e_vc_process_vflr_event(pf); 11353 i40e_watchdog_subtask(pf); 11354 i40e_fdir_reinit_subtask(pf); 11355 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 11356 /* Client subtask will reopen next time through. */ 11357 i40e_notify_client_of_netdev_close(pf, true); 11358 } else { 11359 i40e_client_subtask(pf); 11360 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 11361 pf->state)) 11362 i40e_notify_client_of_l2_param_changes(pf); 11363 } 11364 i40e_sync_filters_subtask(pf); 11365 } else { 11366 i40e_reset_subtask(pf); 11367 } 11368 11369 i40e_clean_adminq_subtask(pf); 11370 11371 /* flush memory to make sure state is correct before next watchdog */ 11372 smp_mb__before_atomic(); 11373 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11374 11375 /* If the tasks have taken longer than one timer cycle or there 11376 * is more work to be done, reschedule the service task now 11377 * rather than wait for the timer to tick again. 11378 */ 11379 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 11380 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 11381 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 11382 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 11383 i40e_service_event_schedule(pf); 11384} 11385 11386/** 11387 * i40e_service_timer - timer callback 11388 * @t: timer list pointer 11389 **/ 11390static void i40e_service_timer(struct timer_list *t) 11391{ 11392 struct i40e_pf *pf = timer_container_of(pf, t, service_timer); 11393 11394 mod_timer(&pf->service_timer, 11395 round_jiffies(jiffies + pf->service_timer_period)); 11396 i40e_service_event_schedule(pf); 11397} 11398 11399/** 11400 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 11401 * @vsi: the VSI being configured 11402 **/ 11403static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 11404{ 11405 struct i40e_pf *pf = vsi->back; 11406 11407 switch (vsi->type) { 11408 case I40E_VSI_MAIN: 11409 vsi->alloc_queue_pairs = pf->num_lan_qps; 11410 if (!vsi->num_tx_desc) 11411 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11412 I40E_REQ_DESCRIPTOR_MULTIPLE); 11413 if (!vsi->num_rx_desc) 11414 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11415 I40E_REQ_DESCRIPTOR_MULTIPLE); 11416 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11417 vsi->num_q_vectors = pf->num_lan_msix; 11418 else 11419 vsi->num_q_vectors = 1; 11420 11421 break; 11422 11423 case I40E_VSI_FDIR: 11424 vsi->alloc_queue_pairs = 1; 11425 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11426 I40E_REQ_DESCRIPTOR_MULTIPLE); 11427 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11428 I40E_REQ_DESCRIPTOR_MULTIPLE); 11429 vsi->num_q_vectors = pf->num_fdsb_msix; 11430 break; 11431 11432 case I40E_VSI_VMDQ2: 11433 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 11434 if (!vsi->num_tx_desc) 11435 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11436 I40E_REQ_DESCRIPTOR_MULTIPLE); 11437 if (!vsi->num_rx_desc) 11438 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11439 I40E_REQ_DESCRIPTOR_MULTIPLE); 11440 vsi->num_q_vectors = pf->num_vmdq_msix; 11441 break; 11442 11443 case I40E_VSI_SRIOV: 11444 vsi->alloc_queue_pairs = pf->num_vf_qps; 11445 if (!vsi->num_tx_desc) 11446 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11447 I40E_REQ_DESCRIPTOR_MULTIPLE); 11448 if (!vsi->num_rx_desc) 11449 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11450 I40E_REQ_DESCRIPTOR_MULTIPLE); 11451 break; 11452 11453 default: 11454 WARN_ON(1); 11455 return -ENODATA; 11456 } 11457 11458 if (is_kdump_kernel()) { 11459 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS; 11460 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS; 11461 } 11462 11463 return 0; 11464} 11465 11466/** 11467 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 11468 * @vsi: VSI pointer 11469 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 11470 * 11471 * On error: returns error code (negative) 11472 * On success: returns 0 11473 **/ 11474static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 11475{ 11476 struct i40e_ring **next_rings; 11477 int size; 11478 int ret = 0; 11479 11480 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 11481 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 11482 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 11483 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 11484 if (!vsi->tx_rings) 11485 return -ENOMEM; 11486 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 11487 if (i40e_enabled_xdp_vsi(vsi)) { 11488 vsi->xdp_rings = next_rings; 11489 next_rings += vsi->alloc_queue_pairs; 11490 } 11491 vsi->rx_rings = next_rings; 11492 11493 if (alloc_qvectors) { 11494 /* allocate memory for q_vector pointers */ 11495 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 11496 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 11497 if (!vsi->q_vectors) { 11498 ret = -ENOMEM; 11499 goto err_vectors; 11500 } 11501 } 11502 return ret; 11503 11504err_vectors: 11505 kfree(vsi->tx_rings); 11506 return ret; 11507} 11508 11509/** 11510 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 11511 * @pf: board private structure 11512 * @type: type of VSI 11513 * 11514 * On error: returns error code (negative) 11515 * On success: returns vsi index in PF (positive) 11516 **/ 11517static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 11518{ 11519 int ret = -ENODEV; 11520 struct i40e_vsi *vsi; 11521 int vsi_idx; 11522 int i; 11523 11524 /* Need to protect the allocation of the VSIs at the PF level */ 11525 mutex_lock(&pf->switch_mutex); 11526 11527 /* VSI list may be fragmented if VSI creation/destruction has 11528 * been happening. We can afford to do a quick scan to look 11529 * for any free VSIs in the list. 11530 * 11531 * find next empty vsi slot, looping back around if necessary 11532 */ 11533 i = pf->next_vsi; 11534 while (i < pf->num_alloc_vsi && pf->vsi[i]) 11535 i++; 11536 if (i >= pf->num_alloc_vsi) { 11537 i = 0; 11538 while (i < pf->next_vsi && pf->vsi[i]) 11539 i++; 11540 } 11541 11542 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 11543 vsi_idx = i; /* Found one! */ 11544 } else { 11545 ret = -ENODEV; 11546 goto unlock_pf; /* out of VSI slots! */ 11547 } 11548 pf->next_vsi = ++i; 11549 11550 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 11551 if (!vsi) { 11552 ret = -ENOMEM; 11553 goto unlock_pf; 11554 } 11555 vsi->type = type; 11556 vsi->back = pf; 11557 set_bit(__I40E_VSI_DOWN, vsi->state); 11558 vsi->flags = 0; 11559 vsi->idx = vsi_idx; 11560 vsi->int_rate_limit = 0; 11561 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 11562 pf->rss_table_size : 64; 11563 vsi->netdev_registered = false; 11564 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 11565 hash_init(vsi->mac_filter_hash); 11566 vsi->irqs_ready = false; 11567 11568 if (type == I40E_VSI_MAIN) { 11569 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 11570 if (!vsi->af_xdp_zc_qps) 11571 goto err_rings; 11572 } 11573 11574 ret = i40e_set_num_rings_in_vsi(vsi); 11575 if (ret) 11576 goto err_rings; 11577 11578 ret = i40e_vsi_alloc_arrays(vsi, true); 11579 if (ret) 11580 goto err_rings; 11581 11582 /* Setup default MSIX irq handler for VSI */ 11583 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 11584 11585 /* Initialize VSI lock */ 11586 spin_lock_init(&vsi->mac_filter_hash_lock); 11587 pf->vsi[vsi_idx] = vsi; 11588 ret = vsi_idx; 11589 goto unlock_pf; 11590 11591err_rings: 11592 bitmap_free(vsi->af_xdp_zc_qps); 11593 pf->next_vsi = i - 1; 11594 kfree(vsi); 11595unlock_pf: 11596 mutex_unlock(&pf->switch_mutex); 11597 return ret; 11598} 11599 11600/** 11601 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 11602 * @vsi: VSI pointer 11603 * @free_qvectors: a bool to specify if q_vectors need to be freed. 11604 * 11605 * On error: returns error code (negative) 11606 * On success: returns 0 11607 **/ 11608static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 11609{ 11610 /* free the ring and vector containers */ 11611 if (free_qvectors) { 11612 kfree(vsi->q_vectors); 11613 vsi->q_vectors = NULL; 11614 } 11615 kfree(vsi->tx_rings); 11616 vsi->tx_rings = NULL; 11617 vsi->rx_rings = NULL; 11618 vsi->xdp_rings = NULL; 11619} 11620 11621/** 11622 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 11623 * and lookup table 11624 * @vsi: Pointer to VSI structure 11625 */ 11626static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 11627{ 11628 if (!vsi) 11629 return; 11630 11631 kfree(vsi->rss_hkey_user); 11632 vsi->rss_hkey_user = NULL; 11633 11634 kfree(vsi->rss_lut_user); 11635 vsi->rss_lut_user = NULL; 11636} 11637 11638/** 11639 * i40e_vsi_clear - Deallocate the VSI provided 11640 * @vsi: the VSI being un-configured 11641 **/ 11642static int i40e_vsi_clear(struct i40e_vsi *vsi) 11643{ 11644 struct i40e_pf *pf; 11645 11646 if (!vsi) 11647 return 0; 11648 11649 if (!vsi->back) 11650 goto free_vsi; 11651 pf = vsi->back; 11652 11653 mutex_lock(&pf->switch_mutex); 11654 if (!pf->vsi[vsi->idx]) { 11655 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 11656 vsi->idx, vsi->idx, vsi->type); 11657 goto unlock_vsi; 11658 } 11659 11660 if (pf->vsi[vsi->idx] != vsi) { 11661 dev_err(&pf->pdev->dev, 11662 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 11663 pf->vsi[vsi->idx]->idx, 11664 pf->vsi[vsi->idx]->type, 11665 vsi->idx, vsi->type); 11666 goto unlock_vsi; 11667 } 11668 11669 /* updates the PF for this cleared vsi */ 11670 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 11671 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 11672 11673 bitmap_free(vsi->af_xdp_zc_qps); 11674 i40e_vsi_free_arrays(vsi, true); 11675 i40e_clear_rss_config_user(vsi); 11676 11677 pf->vsi[vsi->idx] = NULL; 11678 if (vsi->idx < pf->next_vsi) 11679 pf->next_vsi = vsi->idx; 11680 11681unlock_vsi: 11682 mutex_unlock(&pf->switch_mutex); 11683free_vsi: 11684 kfree(vsi); 11685 11686 return 0; 11687} 11688 11689/** 11690 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 11691 * @vsi: the VSI being cleaned 11692 **/ 11693static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 11694{ 11695 int i; 11696 11697 if (vsi->tx_rings && vsi->tx_rings[0]) { 11698 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11699 kfree_rcu(vsi->tx_rings[i], rcu); 11700 WRITE_ONCE(vsi->tx_rings[i], NULL); 11701 WRITE_ONCE(vsi->rx_rings[i], NULL); 11702 if (vsi->xdp_rings) 11703 WRITE_ONCE(vsi->xdp_rings[i], NULL); 11704 } 11705 } 11706} 11707 11708/** 11709 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 11710 * @vsi: the VSI being configured 11711 **/ 11712static int i40e_alloc_rings(struct i40e_vsi *vsi) 11713{ 11714 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 11715 struct i40e_pf *pf = vsi->back; 11716 struct i40e_ring *ring; 11717 11718 /* Set basic values in the rings to be used later during open() */ 11719 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11720 /* allocate space for both Tx and Rx in one shot */ 11721 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 11722 if (!ring) 11723 goto err_out; 11724 11725 ring->queue_index = i; 11726 ring->reg_idx = vsi->base_queue + i; 11727 ring->ring_active = false; 11728 ring->vsi = vsi; 11729 ring->netdev = vsi->netdev; 11730 ring->dev = &pf->pdev->dev; 11731 ring->count = vsi->num_tx_desc; 11732 ring->size = 0; 11733 ring->dcb_tc = 0; 11734 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11735 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11736 ring->itr_setting = pf->tx_itr_default; 11737 WRITE_ONCE(vsi->tx_rings[i], ring++); 11738 11739 if (!i40e_enabled_xdp_vsi(vsi)) 11740 goto setup_rx; 11741 11742 ring->queue_index = vsi->alloc_queue_pairs + i; 11743 ring->reg_idx = vsi->base_queue + ring->queue_index; 11744 ring->ring_active = false; 11745 ring->vsi = vsi; 11746 ring->netdev = NULL; 11747 ring->dev = &pf->pdev->dev; 11748 ring->count = vsi->num_tx_desc; 11749 ring->size = 0; 11750 ring->dcb_tc = 0; 11751 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps)) 11752 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11753 set_ring_xdp(ring); 11754 ring->itr_setting = pf->tx_itr_default; 11755 WRITE_ONCE(vsi->xdp_rings[i], ring++); 11756 11757setup_rx: 11758 ring->queue_index = i; 11759 ring->reg_idx = vsi->base_queue + i; 11760 ring->ring_active = false; 11761 ring->vsi = vsi; 11762 ring->netdev = vsi->netdev; 11763 ring->dev = &pf->pdev->dev; 11764 ring->count = vsi->num_rx_desc; 11765 ring->size = 0; 11766 ring->dcb_tc = 0; 11767 ring->itr_setting = pf->rx_itr_default; 11768 WRITE_ONCE(vsi->rx_rings[i], ring); 11769 } 11770 11771 return 0; 11772 11773err_out: 11774 i40e_vsi_clear_rings(vsi); 11775 return -ENOMEM; 11776} 11777 11778/** 11779 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 11780 * @pf: board private structure 11781 * @vectors: the number of MSI-X vectors to request 11782 * 11783 * Returns the number of vectors reserved, or error 11784 **/ 11785static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 11786{ 11787 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 11788 I40E_MIN_MSIX, vectors); 11789 if (vectors < 0) { 11790 dev_info(&pf->pdev->dev, 11791 "MSI-X vector reservation failed: %d\n", vectors); 11792 vectors = 0; 11793 } 11794 11795 return vectors; 11796} 11797 11798/** 11799 * i40e_init_msix - Setup the MSIX capability 11800 * @pf: board private structure 11801 * 11802 * Work with the OS to set up the MSIX vectors needed. 11803 * 11804 * Returns the number of vectors reserved or negative on failure 11805 **/ 11806static int i40e_init_msix(struct i40e_pf *pf) 11807{ 11808 struct i40e_hw *hw = &pf->hw; 11809 int cpus, extra_vectors; 11810 int vectors_left; 11811 int v_budget, i; 11812 int v_actual; 11813 int iwarp_requested = 0; 11814 11815 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 11816 return -ENODEV; 11817 11818 /* The number of vectors we'll request will be comprised of: 11819 * - Add 1 for "other" cause for Admin Queue events, etc. 11820 * - The number of LAN queue pairs 11821 * - Queues being used for RSS. 11822 * We don't need as many as max_rss_size vectors. 11823 * use rss_size instead in the calculation since that 11824 * is governed by number of cpus in the system. 11825 * - assumes symmetric Tx/Rx pairing 11826 * - The number of VMDq pairs 11827 * - The CPU count within the NUMA node if iWARP is enabled 11828 * Once we count this up, try the request. 11829 * 11830 * If we can't get what we want, we'll simplify to nearly nothing 11831 * and try again. If that still fails, we punt. 11832 */ 11833 vectors_left = hw->func_caps.num_msix_vectors; 11834 v_budget = 0; 11835 11836 /* reserve one vector for miscellaneous handler */ 11837 if (vectors_left) { 11838 v_budget++; 11839 vectors_left--; 11840 } 11841 11842 /* reserve some vectors for the main PF traffic queues. Initially we 11843 * only reserve at most 50% of the available vectors, in the case that 11844 * the number of online CPUs is large. This ensures that we can enable 11845 * extra features as well. Once we've enabled the other features, we 11846 * will use any remaining vectors to reach as close as we can to the 11847 * number of online CPUs. 11848 */ 11849 cpus = num_online_cpus(); 11850 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 11851 vectors_left -= pf->num_lan_msix; 11852 11853 /* reserve one vector for sideband flow director */ 11854 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11855 if (vectors_left) { 11856 pf->num_fdsb_msix = 1; 11857 v_budget++; 11858 vectors_left--; 11859 } else { 11860 pf->num_fdsb_msix = 0; 11861 } 11862 } 11863 11864 /* can we reserve enough for iWARP? */ 11865 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11866 iwarp_requested = pf->num_iwarp_msix; 11867 11868 if (!vectors_left) 11869 pf->num_iwarp_msix = 0; 11870 else if (vectors_left < pf->num_iwarp_msix) 11871 pf->num_iwarp_msix = 1; 11872 v_budget += pf->num_iwarp_msix; 11873 vectors_left -= pf->num_iwarp_msix; 11874 } 11875 11876 /* any vectors left over go for VMDq support */ 11877 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) { 11878 if (!vectors_left) { 11879 pf->num_vmdq_msix = 0; 11880 pf->num_vmdq_qps = 0; 11881 } else { 11882 int vmdq_vecs_wanted = 11883 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11884 int vmdq_vecs = 11885 min_t(int, vectors_left, vmdq_vecs_wanted); 11886 11887 /* if we're short on vectors for what's desired, we limit 11888 * the queues per vmdq. If this is still more than are 11889 * available, the user will need to change the number of 11890 * queues/vectors used by the PF later with the ethtool 11891 * channels command 11892 */ 11893 if (vectors_left < vmdq_vecs_wanted) { 11894 pf->num_vmdq_qps = 1; 11895 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11896 vmdq_vecs = min_t(int, 11897 vectors_left, 11898 vmdq_vecs_wanted); 11899 } 11900 pf->num_vmdq_msix = pf->num_vmdq_qps; 11901 11902 v_budget += vmdq_vecs; 11903 vectors_left -= vmdq_vecs; 11904 } 11905 } 11906 11907 /* On systems with a large number of SMP cores, we previously limited 11908 * the number of vectors for num_lan_msix to be at most 50% of the 11909 * available vectors, to allow for other features. Now, we add back 11910 * the remaining vectors. However, we ensure that the total 11911 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11912 * calculate the number of vectors we can add without going over the 11913 * cap of CPUs. For systems with a small number of CPUs this will be 11914 * zero. 11915 */ 11916 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11917 pf->num_lan_msix += extra_vectors; 11918 vectors_left -= extra_vectors; 11919 11920 WARN(vectors_left < 0, 11921 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11922 11923 v_budget += pf->num_lan_msix; 11924 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11925 GFP_KERNEL); 11926 if (!pf->msix_entries) 11927 return -ENOMEM; 11928 11929 for (i = 0; i < v_budget; i++) 11930 pf->msix_entries[i].entry = i; 11931 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11932 11933 if (v_actual < I40E_MIN_MSIX) { 11934 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 11935 kfree(pf->msix_entries); 11936 pf->msix_entries = NULL; 11937 pci_disable_msix(pf->pdev); 11938 return -ENODEV; 11939 11940 } else if (v_actual == I40E_MIN_MSIX) { 11941 /* Adjust for minimal MSIX use */ 11942 pf->num_vmdq_vsis = 0; 11943 pf->num_vmdq_qps = 0; 11944 pf->num_lan_qps = 1; 11945 pf->num_lan_msix = 1; 11946 11947 } else if (v_actual != v_budget) { 11948 /* If we have limited resources, we will start with no vectors 11949 * for the special features and then allocate vectors to some 11950 * of these features based on the policy and at the end disable 11951 * the features that did not get any vectors. 11952 */ 11953 int vec; 11954 11955 dev_info(&pf->pdev->dev, 11956 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11957 v_actual, v_budget); 11958 /* reserve the misc vector */ 11959 vec = v_actual - 1; 11960 11961 /* Scale vector usage down */ 11962 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11963 pf->num_vmdq_vsis = 1; 11964 pf->num_vmdq_qps = 1; 11965 11966 /* partition out the remaining vectors */ 11967 switch (vec) { 11968 case 2: 11969 pf->num_lan_msix = 1; 11970 break; 11971 case 3: 11972 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11973 pf->num_lan_msix = 1; 11974 pf->num_iwarp_msix = 1; 11975 } else { 11976 pf->num_lan_msix = 2; 11977 } 11978 break; 11979 default: 11980 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 11981 pf->num_iwarp_msix = min_t(int, (vec / 3), 11982 iwarp_requested); 11983 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11984 I40E_DEFAULT_NUM_VMDQ_VSI); 11985 } else { 11986 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11987 I40E_DEFAULT_NUM_VMDQ_VSI); 11988 } 11989 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 11990 pf->num_fdsb_msix = 1; 11991 vec--; 11992 } 11993 pf->num_lan_msix = min_t(int, 11994 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11995 pf->num_lan_msix); 11996 pf->num_lan_qps = pf->num_lan_msix; 11997 break; 11998 } 11999 } 12000 12001 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) { 12002 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 12003 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12004 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12005 } 12006 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) { 12007 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 12008 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12009 } 12010 12011 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) && 12012 pf->num_iwarp_msix == 0) { 12013 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 12014 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12015 } 12016 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 12017 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 12018 pf->num_lan_msix, 12019 pf->num_vmdq_msix * pf->num_vmdq_vsis, 12020 pf->num_fdsb_msix, 12021 pf->num_iwarp_msix); 12022 12023 return v_actual; 12024} 12025 12026/** 12027 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 12028 * @vsi: the VSI being configured 12029 * @v_idx: index of the vector in the vsi struct 12030 * 12031 * We allocate one q_vector. If allocation fails we return -ENOMEM. 12032 **/ 12033static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 12034{ 12035 struct i40e_q_vector *q_vector; 12036 12037 /* allocate q_vector */ 12038 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 12039 if (!q_vector) 12040 return -ENOMEM; 12041 12042 q_vector->vsi = vsi; 12043 q_vector->v_idx = v_idx; 12044 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 12045 12046 if (vsi->netdev) 12047 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll); 12048 12049 /* tie q_vector and vsi together */ 12050 vsi->q_vectors[v_idx] = q_vector; 12051 12052 return 0; 12053} 12054 12055/** 12056 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 12057 * @vsi: the VSI being configured 12058 * 12059 * We allocate one q_vector per queue interrupt. If allocation fails we 12060 * return -ENOMEM. 12061 **/ 12062static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 12063{ 12064 struct i40e_pf *pf = vsi->back; 12065 int err, v_idx, num_q_vectors; 12066 12067 /* if not MSIX, give the one vector only to the LAN VSI */ 12068 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12069 num_q_vectors = vsi->num_q_vectors; 12070 else if (vsi->type == I40E_VSI_MAIN) 12071 num_q_vectors = 1; 12072 else 12073 return -EINVAL; 12074 12075 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 12076 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 12077 if (err) 12078 goto err_out; 12079 } 12080 12081 return 0; 12082 12083err_out: 12084 while (v_idx--) 12085 i40e_free_q_vector(vsi, v_idx); 12086 12087 return err; 12088} 12089 12090/** 12091 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 12092 * @pf: board private structure to initialize 12093 **/ 12094static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 12095{ 12096 int vectors = 0; 12097 ssize_t size; 12098 12099 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12100 vectors = i40e_init_msix(pf); 12101 if (vectors < 0) { 12102 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12103 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12104 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 12105 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 12106 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 12107 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12108 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12109 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12110 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12111 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12112 12113 /* rework the queue expectations without MSIX */ 12114 i40e_determine_queue_usage(pf); 12115 } 12116 } 12117 12118 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 12119 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) { 12120 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 12121 vectors = pci_enable_msi(pf->pdev); 12122 if (vectors < 0) { 12123 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 12124 vectors); 12125 clear_bit(I40E_FLAG_MSI_ENA, pf->flags); 12126 } 12127 vectors = 1; /* one MSI or Legacy vector */ 12128 } 12129 12130 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) && 12131 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 12132 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 12133 12134 /* set up vector assignment tracking */ 12135 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 12136 pf->irq_pile = kzalloc(size, GFP_KERNEL); 12137 if (!pf->irq_pile) 12138 return -ENOMEM; 12139 12140 pf->irq_pile->num_entries = vectors; 12141 12142 /* track first vector for misc interrupts, ignore return */ 12143 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 12144 12145 return 0; 12146} 12147 12148/** 12149 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 12150 * @pf: private board data structure 12151 * 12152 * Restore the interrupt scheme that was cleared when we suspended the 12153 * device. This should be called during resume to re-allocate the q_vectors 12154 * and reacquire IRQs. 12155 */ 12156static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 12157{ 12158 struct i40e_vsi *vsi; 12159 int err, i; 12160 12161 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 12162 * scheme. We need to re-enabled them here in order to attempt to 12163 * re-acquire the MSI or MSI-X vectors 12164 */ 12165 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12166 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12167 12168 err = i40e_init_interrupt_scheme(pf); 12169 if (err) 12170 return err; 12171 12172 /* Now that we've re-acquired IRQs, we need to remap the vectors and 12173 * rings together again. 12174 */ 12175 i40e_pf_for_each_vsi(pf, i, vsi) { 12176 err = i40e_vsi_alloc_q_vectors(vsi); 12177 if (err) 12178 goto err_unwind; 12179 12180 i40e_vsi_map_rings_to_vectors(vsi); 12181 } 12182 12183 err = i40e_setup_misc_vector(pf); 12184 if (err) 12185 goto err_unwind; 12186 12187 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) 12188 i40e_client_update_msix_info(pf); 12189 12190 return 0; 12191 12192err_unwind: 12193 while (i--) { 12194 if (pf->vsi[i]) 12195 i40e_vsi_free_q_vectors(pf->vsi[i]); 12196 } 12197 12198 return err; 12199} 12200 12201/** 12202 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 12203 * non queue events in recovery mode 12204 * @pf: board private structure 12205 * 12206 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 12207 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 12208 * This is handled differently than in recovery mode since no Tx/Rx resources 12209 * are being allocated. 12210 **/ 12211static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 12212{ 12213 int err; 12214 12215 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 12216 err = i40e_setup_misc_vector(pf); 12217 12218 if (err) { 12219 dev_info(&pf->pdev->dev, 12220 "MSI-X misc vector request failed, error %d\n", 12221 err); 12222 return err; 12223 } 12224 } else { 12225 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED; 12226 12227 err = request_irq(pf->pdev->irq, i40e_intr, flags, 12228 pf->int_name, pf); 12229 12230 if (err) { 12231 dev_info(&pf->pdev->dev, 12232 "MSI/legacy misc vector request failed, error %d\n", 12233 err); 12234 return err; 12235 } 12236 i40e_enable_misc_int_causes(pf); 12237 i40e_irq_dynamic_enable_icr0(pf); 12238 } 12239 12240 return 0; 12241} 12242 12243/** 12244 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 12245 * @pf: board private structure 12246 * 12247 * This sets up the handler for MSIX 0, which is used to manage the 12248 * non-queue interrupts, e.g. AdminQ and errors. This is not used 12249 * when in MSI or Legacy interrupt mode. 12250 **/ 12251static int i40e_setup_misc_vector(struct i40e_pf *pf) 12252{ 12253 struct i40e_hw *hw = &pf->hw; 12254 int err = 0; 12255 12256 /* Only request the IRQ once, the first time through. */ 12257 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 12258 err = request_irq(pf->msix_entries[0].vector, 12259 i40e_intr, 0, pf->int_name, pf); 12260 if (err) { 12261 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 12262 dev_info(&pf->pdev->dev, 12263 "request_irq for %s failed: %d\n", 12264 pf->int_name, err); 12265 return -EFAULT; 12266 } 12267 } 12268 12269 i40e_enable_misc_int_causes(pf); 12270 12271 /* associate no queues to the misc vector */ 12272 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 12273 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 12274 12275 i40e_flush(hw); 12276 12277 i40e_irq_dynamic_enable_icr0(pf); 12278 12279 return err; 12280} 12281 12282/** 12283 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 12284 * @vsi: Pointer to vsi structure 12285 * @seed: Buffter to store the hash keys 12286 * @lut: Buffer to store the lookup table entries 12287 * @lut_size: Size of buffer to store the lookup table entries 12288 * 12289 * Return 0 on success, negative on failure 12290 */ 12291static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 12292 u8 *lut, u16 lut_size) 12293{ 12294 struct i40e_pf *pf = vsi->back; 12295 struct i40e_hw *hw = &pf->hw; 12296 int ret = 0; 12297 12298 if (seed) { 12299 ret = i40e_aq_get_rss_key(hw, vsi->id, 12300 (struct i40e_aqc_get_set_rss_key_data *)seed); 12301 if (ret) { 12302 dev_info(&pf->pdev->dev, 12303 "Cannot get RSS key, err %pe aq_err %s\n", 12304 ERR_PTR(ret), 12305 libie_aq_str(pf->hw.aq.asq_last_status)); 12306 return ret; 12307 } 12308 } 12309 12310 if (lut) { 12311 bool pf_lut = vsi->type == I40E_VSI_MAIN; 12312 12313 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 12314 if (ret) { 12315 dev_info(&pf->pdev->dev, 12316 "Cannot get RSS lut, err %pe aq_err %s\n", 12317 ERR_PTR(ret), 12318 libie_aq_str(pf->hw.aq.asq_last_status)); 12319 return ret; 12320 } 12321 } 12322 12323 return ret; 12324} 12325 12326/** 12327 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 12328 * @vsi: Pointer to vsi structure 12329 * @seed: RSS hash seed 12330 * @lut: Lookup table 12331 * @lut_size: Lookup table size 12332 * 12333 * Returns 0 on success, negative on failure 12334 **/ 12335static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 12336 const u8 *lut, u16 lut_size) 12337{ 12338 struct i40e_pf *pf = vsi->back; 12339 struct i40e_hw *hw = &pf->hw; 12340 u16 vf_id = vsi->vf_id; 12341 u8 i; 12342 12343 /* Fill out hash function seed */ 12344 if (seed) { 12345 u32 *seed_dw = (u32 *)seed; 12346 12347 if (vsi->type == I40E_VSI_MAIN) { 12348 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12349 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 12350 } else if (vsi->type == I40E_VSI_SRIOV) { 12351 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 12352 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 12353 } else { 12354 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 12355 } 12356 } 12357 12358 if (lut) { 12359 u32 *lut_dw = (u32 *)lut; 12360 12361 if (vsi->type == I40E_VSI_MAIN) { 12362 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12363 return -EINVAL; 12364 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12365 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 12366 } else if (vsi->type == I40E_VSI_SRIOV) { 12367 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 12368 return -EINVAL; 12369 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12370 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 12371 } else { 12372 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12373 } 12374 } 12375 i40e_flush(hw); 12376 12377 return 0; 12378} 12379 12380/** 12381 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 12382 * @vsi: Pointer to VSI structure 12383 * @seed: Buffer to store the keys 12384 * @lut: Buffer to store the lookup table entries 12385 * @lut_size: Size of buffer to store the lookup table entries 12386 * 12387 * Returns 0 on success, negative on failure 12388 */ 12389static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 12390 u8 *lut, u16 lut_size) 12391{ 12392 struct i40e_pf *pf = vsi->back; 12393 struct i40e_hw *hw = &pf->hw; 12394 u16 i; 12395 12396 if (seed) { 12397 u32 *seed_dw = (u32 *)seed; 12398 12399 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12400 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 12401 } 12402 if (lut) { 12403 u32 *lut_dw = (u32 *)lut; 12404 12405 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12406 return -EINVAL; 12407 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12408 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 12409 } 12410 12411 return 0; 12412} 12413 12414/** 12415 * i40e_config_rss - Configure RSS keys and lut 12416 * @vsi: Pointer to VSI structure 12417 * @seed: RSS hash seed 12418 * @lut: Lookup table 12419 * @lut_size: Lookup table size 12420 * 12421 * Returns 0 on success, negative on failure 12422 */ 12423int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12424{ 12425 struct i40e_pf *pf = vsi->back; 12426 12427 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12428 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 12429 else 12430 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 12431} 12432 12433/** 12434 * i40e_get_rss - Get RSS keys and lut 12435 * @vsi: Pointer to VSI structure 12436 * @seed: Buffer to store the keys 12437 * @lut: Buffer to store the lookup table entries 12438 * @lut_size: Size of buffer to store the lookup table entries 12439 * 12440 * Returns 0 on success, negative on failure 12441 */ 12442int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12443{ 12444 struct i40e_pf *pf = vsi->back; 12445 12446 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps)) 12447 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 12448 else 12449 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 12450} 12451 12452/** 12453 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 12454 * @pf: Pointer to board private structure 12455 * @lut: Lookup table 12456 * @rss_table_size: Lookup table size 12457 * @rss_size: Range of queue number for hashing 12458 */ 12459void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 12460 u16 rss_table_size, u16 rss_size) 12461{ 12462 u16 i; 12463 12464 for (i = 0; i < rss_table_size; i++) 12465 lut[i] = i % rss_size; 12466} 12467 12468/** 12469 * i40e_pf_config_rss - Prepare for RSS if used 12470 * @pf: board private structure 12471 **/ 12472static int i40e_pf_config_rss(struct i40e_pf *pf) 12473{ 12474 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12475 u8 seed[I40E_HKEY_ARRAY_SIZE]; 12476 u8 *lut; 12477 struct i40e_hw *hw = &pf->hw; 12478 u32 reg_val; 12479 u64 hena; 12480 int ret; 12481 12482 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 12483 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 12484 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 12485 hena |= i40e_pf_get_default_rss_hashcfg(pf); 12486 12487 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 12488 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 12489 12490 /* Determine the RSS table size based on the hardware capabilities */ 12491 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 12492 reg_val = (pf->rss_table_size == 512) ? 12493 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 12494 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 12495 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 12496 12497 /* Determine the RSS size of the VSI */ 12498 if (!vsi->rss_size) { 12499 u16 qcount; 12500 /* If the firmware does something weird during VSI init, we 12501 * could end up with zero TCs. Check for that to avoid 12502 * divide-by-zero. It probably won't pass traffic, but it also 12503 * won't panic. 12504 */ 12505 qcount = vsi->num_queue_pairs / 12506 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 12507 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12508 } 12509 if (!vsi->rss_size) 12510 return -EINVAL; 12511 12512 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 12513 if (!lut) 12514 return -ENOMEM; 12515 12516 /* Use user configured lut if there is one, otherwise use default */ 12517 if (vsi->rss_lut_user) 12518 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 12519 else 12520 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 12521 12522 /* Use user configured hash key if there is one, otherwise 12523 * use default. 12524 */ 12525 if (vsi->rss_hkey_user) 12526 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 12527 else 12528 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 12529 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 12530 kfree(lut); 12531 12532 return ret; 12533} 12534 12535/** 12536 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 12537 * @pf: board private structure 12538 * @queue_count: the requested queue count for rss. 12539 * 12540 * returns 0 if rss is not enabled, if enabled returns the final rss queue 12541 * count which may be different from the requested queue count. 12542 * Note: expects to be called while under rtnl_lock() 12543 **/ 12544int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 12545{ 12546 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf); 12547 int new_rss_size; 12548 12549 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 12550 return 0; 12551 12552 queue_count = min_t(int, queue_count, num_online_cpus()); 12553 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 12554 12555 if (queue_count != vsi->num_queue_pairs) { 12556 u16 qcount; 12557 12558 vsi->req_queue_pairs = queue_count; 12559 i40e_prep_for_reset(pf); 12560 if (test_bit(__I40E_IN_REMOVE, pf->state)) 12561 return pf->alloc_rss_size; 12562 12563 pf->alloc_rss_size = new_rss_size; 12564 12565 i40e_reset_and_rebuild(pf, true, true); 12566 12567 /* Discard the user configured hash keys and lut, if less 12568 * queues are enabled. 12569 */ 12570 if (queue_count < vsi->rss_size) { 12571 i40e_clear_rss_config_user(vsi); 12572 dev_dbg(&pf->pdev->dev, 12573 "discard user configured hash keys and lut\n"); 12574 } 12575 12576 /* Reset vsi->rss_size, as number of enabled queues changed */ 12577 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 12578 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12579 12580 i40e_pf_config_rss(pf); 12581 } 12582 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 12583 vsi->req_queue_pairs, pf->rss_size_max); 12584 return pf->alloc_rss_size; 12585} 12586 12587/** 12588 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 12589 * @pf: board private structure 12590 **/ 12591int i40e_get_partition_bw_setting(struct i40e_pf *pf) 12592{ 12593 bool min_valid, max_valid; 12594 u32 max_bw, min_bw; 12595 int status; 12596 12597 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 12598 &min_valid, &max_valid); 12599 12600 if (!status) { 12601 if (min_valid) 12602 pf->min_bw = min_bw; 12603 if (max_valid) 12604 pf->max_bw = max_bw; 12605 } 12606 12607 return status; 12608} 12609 12610/** 12611 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 12612 * @pf: board private structure 12613 **/ 12614int i40e_set_partition_bw_setting(struct i40e_pf *pf) 12615{ 12616 struct i40e_aqc_configure_partition_bw_data bw_data; 12617 int status; 12618 12619 memset(&bw_data, 0, sizeof(bw_data)); 12620 12621 /* Set the valid bit for this PF */ 12622 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 12623 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 12624 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 12625 12626 /* Set the new bandwidths */ 12627 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 12628 12629 return status; 12630} 12631 12632/** 12633 * i40e_is_total_port_shutdown_enabled - read NVM and return value 12634 * if total port shutdown feature is enabled for this PF 12635 * @pf: board private structure 12636 **/ 12637static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 12638{ 12639#define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 12640#define I40E_FEATURES_ENABLE_PTR 0x2A 12641#define I40E_CURRENT_SETTING_PTR 0x2B 12642#define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 12643#define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 12644#define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 12645#define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 12646 u16 sr_emp_sr_settings_ptr = 0; 12647 u16 features_enable = 0; 12648 u16 link_behavior = 0; 12649 int read_status = 0; 12650 bool ret = false; 12651 12652 read_status = i40e_read_nvm_word(&pf->hw, 12653 I40E_SR_EMP_SR_SETTINGS_PTR, 12654 &sr_emp_sr_settings_ptr); 12655 if (read_status) 12656 goto err_nvm; 12657 read_status = i40e_read_nvm_word(&pf->hw, 12658 sr_emp_sr_settings_ptr + 12659 I40E_FEATURES_ENABLE_PTR, 12660 &features_enable); 12661 if (read_status) 12662 goto err_nvm; 12663 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 12664 read_status = i40e_read_nvm_module_data(&pf->hw, 12665 I40E_SR_EMP_SR_SETTINGS_PTR, 12666 I40E_CURRENT_SETTING_PTR, 12667 I40E_LINK_BEHAVIOR_WORD_OFFSET, 12668 I40E_LINK_BEHAVIOR_WORD_LENGTH, 12669 &link_behavior); 12670 if (read_status) 12671 goto err_nvm; 12672 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 12673 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 12674 } 12675 return ret; 12676 12677err_nvm: 12678 dev_warn(&pf->pdev->dev, 12679 "total-port-shutdown feature is off due to read nvm error: %pe\n", 12680 ERR_PTR(read_status)); 12681 return ret; 12682} 12683 12684/** 12685 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 12686 * @pf: board private structure to initialize 12687 * 12688 * i40e_sw_init initializes the Adapter private data structure. 12689 * Fields are initialized based on PCI device information and 12690 * OS network device settings (MTU size). 12691 **/ 12692static int i40e_sw_init(struct i40e_pf *pf) 12693{ 12694 int err = 0; 12695 int size; 12696 u16 pow; 12697 12698 /* Set default capability flags */ 12699 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS); 12700 set_bit(I40E_FLAG_MSI_ENA, pf->flags); 12701 set_bit(I40E_FLAG_MSIX_ENA, pf->flags); 12702 12703 /* Set default ITR */ 12704 pf->rx_itr_default = I40E_ITR_RX_DEF; 12705 pf->tx_itr_default = I40E_ITR_TX_DEF; 12706 12707 /* Depending on PF configurations, it is possible that the RSS 12708 * maximum might end up larger than the available queues 12709 */ 12710 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 12711 pf->alloc_rss_size = 1; 12712 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 12713 pf->rss_size_max = min_t(int, pf->rss_size_max, 12714 pf->hw.func_caps.num_tx_qp); 12715 12716 /* find the next higher power-of-2 of num cpus */ 12717 pow = roundup_pow_of_two(num_online_cpus()); 12718 pf->rss_size_max = min_t(int, pf->rss_size_max, pow); 12719 12720 if (pf->hw.func_caps.rss) { 12721 set_bit(I40E_FLAG_RSS_ENA, pf->flags); 12722 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 12723 num_online_cpus()); 12724 } 12725 12726 /* MFP mode enabled */ 12727 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 12728 set_bit(I40E_FLAG_MFP_ENA, pf->flags); 12729 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 12730 if (i40e_get_partition_bw_setting(pf)) { 12731 dev_warn(&pf->pdev->dev, 12732 "Could not get partition bw settings\n"); 12733 } else { 12734 dev_info(&pf->pdev->dev, 12735 "Partition BW Min = %8.8x, Max = %8.8x\n", 12736 pf->min_bw, pf->max_bw); 12737 12738 /* nudge the Tx scheduler */ 12739 i40e_set_partition_bw_setting(pf); 12740 } 12741 } 12742 12743 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 12744 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 12745 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 12746 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 12747 pf->hw.num_partitions > 1) 12748 dev_info(&pf->pdev->dev, 12749 "Flow Director Sideband mode Disabled in MFP mode\n"); 12750 else 12751 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12752 pf->fdir_pf_filter_count = 12753 pf->hw.func_caps.fd_filters_guaranteed; 12754 pf->hw.fdir_shared_filter_count = 12755 pf->hw.func_caps.fd_filters_best_effort; 12756 } 12757 12758 /* Enable HW ATR eviction if possible */ 12759 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps)) 12760 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags); 12761 12762 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 12763 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 12764 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 12765 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 12766 } 12767 12768 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 12769 set_bit(I40E_FLAG_IWARP_ENA, pf->flags); 12770 /* IWARP needs one extra vector for CQP just like MISC.*/ 12771 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 12772 } 12773 /* Stopping FW LLDP engine is supported on XL710 and X722 12774 * starting from FW versions determined in i40e_init_adminq. 12775 * Stopping the FW LLDP engine is not supported on XL710 12776 * if NPAR is functioning so unset this hw flag in this case. 12777 */ 12778 if (pf->hw.mac.type == I40E_MAC_XL710 && 12779 pf->hw.func_caps.npar_enable) 12780 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps); 12781 12782#ifdef CONFIG_PCI_IOV 12783 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 12784 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 12785 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 12786 pf->num_req_vfs = min_t(int, 12787 pf->hw.func_caps.num_vfs, 12788 I40E_MAX_VF_COUNT); 12789 } 12790#endif /* CONFIG_PCI_IOV */ 12791 pf->lan_veb = I40E_NO_VEB; 12792 pf->lan_vsi = I40E_NO_VSI; 12793 12794 /* By default FW has this off for performance reasons */ 12795 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 12796 12797 /* set up queue assignment tracking */ 12798 size = sizeof(struct i40e_lump_tracking) 12799 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12800 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12801 if (!pf->qp_pile) { 12802 err = -ENOMEM; 12803 goto sw_init_done; 12804 } 12805 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12806 12807 pf->tx_timeout_recovery_level = 1; 12808 12809 if (pf->hw.mac.type != I40E_MAC_X722 && 12810 i40e_is_total_port_shutdown_enabled(pf)) { 12811 /* Link down on close must be on when total port shutdown 12812 * is enabled for a given port 12813 */ 12814 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 12815 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 12816 dev_info(&pf->pdev->dev, 12817 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12818 } 12819 mutex_init(&pf->switch_mutex); 12820 12821sw_init_done: 12822 return err; 12823} 12824 12825/** 12826 * i40e_set_ntuple - set the ntuple feature flag and take action 12827 * @pf: board private structure to initialize 12828 * @features: the feature set that the stack is suggesting 12829 * 12830 * returns a bool to indicate if reset needs to happen 12831 **/ 12832bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12833{ 12834 bool need_reset = false; 12835 12836 /* Check if Flow Director n-tuple support was enabled or disabled. If 12837 * the state changed, we need to reset. 12838 */ 12839 if (features & NETIF_F_NTUPLE) { 12840 /* Enable filters and mark for reset */ 12841 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) 12842 need_reset = true; 12843 /* enable FD_SB only if there is MSI-X vector and no cloud 12844 * filters exist 12845 */ 12846 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12847 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12848 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12849 } 12850 } else { 12851 /* turn off filters, mark for reset and clear SW filter list */ 12852 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 12853 need_reset = true; 12854 i40e_fdir_filter_exit(pf); 12855 } 12856 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 12857 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12858 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 12859 12860 /* reset fd counters */ 12861 pf->fd_add_err = 0; 12862 pf->fd_atr_cnt = 0; 12863 /* if ATR was auto disabled it can be re-enabled. */ 12864 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12865 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 12866 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12867 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12868 } 12869 return need_reset; 12870} 12871 12872/** 12873 * i40e_clear_rss_lut - clear the rx hash lookup table 12874 * @vsi: the VSI being configured 12875 **/ 12876static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12877{ 12878 struct i40e_pf *pf = vsi->back; 12879 struct i40e_hw *hw = &pf->hw; 12880 u16 vf_id = vsi->vf_id; 12881 u8 i; 12882 12883 if (vsi->type == I40E_VSI_MAIN) { 12884 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12885 wr32(hw, I40E_PFQF_HLUT(i), 0); 12886 } else if (vsi->type == I40E_VSI_SRIOV) { 12887 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12888 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12889 } else { 12890 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12891 } 12892} 12893 12894/** 12895 * i40e_set_loopback - turn on/off loopback mode on underlying PF 12896 * @vsi: ptr to VSI 12897 * @ena: flag to indicate the on/off setting 12898 */ 12899static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena) 12900{ 12901 bool if_running = netif_running(vsi->netdev) && 12902 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state); 12903 int ret; 12904 12905 if (if_running) 12906 i40e_down(vsi); 12907 12908 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 12909 if (ret) 12910 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 12911 if (if_running) 12912 i40e_up(vsi); 12913 12914 return ret; 12915} 12916 12917/** 12918 * i40e_set_features - set the netdev feature flags 12919 * @netdev: ptr to the netdev being adjusted 12920 * @features: the feature set that the stack is suggesting 12921 * Note: expects to be called while under rtnl_lock() 12922 **/ 12923static int i40e_set_features(struct net_device *netdev, 12924 netdev_features_t features) 12925{ 12926 struct i40e_netdev_priv *np = netdev_priv(netdev); 12927 struct i40e_vsi *vsi = np->vsi; 12928 struct i40e_pf *pf = vsi->back; 12929 bool need_reset; 12930 12931 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12932 i40e_pf_config_rss(pf); 12933 else if (!(features & NETIF_F_RXHASH) && 12934 netdev->features & NETIF_F_RXHASH) 12935 i40e_clear_rss_lut(vsi); 12936 12937 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12938 i40e_vlan_stripping_enable(vsi); 12939 else 12940 i40e_vlan_stripping_disable(vsi); 12941 12942 if (!(features & NETIF_F_HW_TC) && 12943 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12944 dev_err(&pf->pdev->dev, 12945 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12946 return -EINVAL; 12947 } 12948 12949 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12950 i40e_del_all_macvlans(vsi); 12951 12952 need_reset = i40e_set_ntuple(pf, features); 12953 12954 if (need_reset) 12955 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12956 12957 if ((features ^ netdev->features) & NETIF_F_LOOPBACK) 12958 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 12959 12960 return 0; 12961} 12962 12963static int i40e_udp_tunnel_set_port(struct net_device *netdev, 12964 unsigned int table, unsigned int idx, 12965 struct udp_tunnel_info *ti) 12966{ 12967 struct i40e_netdev_priv *np = netdev_priv(netdev); 12968 struct i40e_hw *hw = &np->vsi->back->hw; 12969 u8 type, filter_index; 12970 int ret; 12971 12972 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : 12973 I40E_AQC_TUNNEL_TYPE_NGE; 12974 12975 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, 12976 NULL); 12977 if (ret) { 12978 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n", 12979 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 12980 return -EIO; 12981 } 12982 12983 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); 12984 return 0; 12985} 12986 12987static int i40e_udp_tunnel_unset_port(struct net_device *netdev, 12988 unsigned int table, unsigned int idx, 12989 struct udp_tunnel_info *ti) 12990{ 12991 struct i40e_netdev_priv *np = netdev_priv(netdev); 12992 struct i40e_hw *hw = &np->vsi->back->hw; 12993 int ret; 12994 12995 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); 12996 if (ret) { 12997 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n", 12998 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 12999 return -EIO; 13000 } 13001 13002 return 0; 13003} 13004 13005static int i40e_get_phys_port_id(struct net_device *netdev, 13006 struct netdev_phys_item_id *ppid) 13007{ 13008 struct i40e_netdev_priv *np = netdev_priv(netdev); 13009 struct i40e_pf *pf = np->vsi->back; 13010 struct i40e_hw *hw = &pf->hw; 13011 13012 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps)) 13013 return -EOPNOTSUPP; 13014 13015 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 13016 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 13017 13018 return 0; 13019} 13020 13021/** 13022 * i40e_ndo_fdb_add - add an entry to the hardware database 13023 * @ndm: the input from the stack 13024 * @tb: pointer to array of nladdr (unused) 13025 * @dev: the net device pointer 13026 * @addr: the MAC address entry being added 13027 * @vid: VLAN ID 13028 * @flags: instructions from stack about fdb operation 13029 * @notified: whether notification was emitted 13030 * @extack: netlink extended ack, unused currently 13031 */ 13032static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 13033 struct net_device *dev, 13034 const unsigned char *addr, u16 vid, 13035 u16 flags, bool *notified, 13036 struct netlink_ext_ack *extack) 13037{ 13038 struct i40e_netdev_priv *np = netdev_priv(dev); 13039 struct i40e_pf *pf = np->vsi->back; 13040 int err = 0; 13041 13042 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) 13043 return -EOPNOTSUPP; 13044 13045 if (vid) { 13046 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 13047 return -EINVAL; 13048 } 13049 13050 /* Hardware does not support aging addresses so if a 13051 * ndm_state is given only allow permanent addresses 13052 */ 13053 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 13054 netdev_info(dev, "FDB only supports static addresses\n"); 13055 return -EINVAL; 13056 } 13057 13058 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 13059 err = dev_uc_add_excl(dev, addr); 13060 else if (is_multicast_ether_addr(addr)) 13061 err = dev_mc_add_excl(dev, addr); 13062 else 13063 err = -EINVAL; 13064 13065 /* Only return duplicate errors if NLM_F_EXCL is set */ 13066 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 13067 err = 0; 13068 13069 return err; 13070} 13071 13072/** 13073 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 13074 * @dev: the netdev being configured 13075 * @nlh: RTNL message 13076 * @flags: bridge flags 13077 * @extack: netlink extended ack 13078 * 13079 * Inserts a new hardware bridge if not already created and 13080 * enables the bridging mode requested (VEB or VEPA). If the 13081 * hardware bridge has already been inserted and the request 13082 * is to change the mode then that requires a PF reset to 13083 * allow rebuild of the components with required hardware 13084 * bridge mode enabled. 13085 * 13086 * Note: expects to be called while under rtnl_lock() 13087 **/ 13088static int i40e_ndo_bridge_setlink(struct net_device *dev, 13089 struct nlmsghdr *nlh, 13090 u16 flags, 13091 struct netlink_ext_ack *extack) 13092{ 13093 struct i40e_netdev_priv *np = netdev_priv(dev); 13094 struct i40e_vsi *vsi = np->vsi; 13095 struct i40e_pf *pf = vsi->back; 13096 struct nlattr *attr, *br_spec; 13097 struct i40e_veb *veb; 13098 int rem; 13099 13100 /* Only for PF VSI for now */ 13101 if (vsi->type != I40E_VSI_MAIN) 13102 return -EOPNOTSUPP; 13103 13104 /* Find the HW bridge for PF VSI */ 13105 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13106 13107 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13108 if (!br_spec) 13109 return -EINVAL; 13110 13111 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 13112 __u16 mode = nla_get_u16(attr); 13113 13114 if ((mode != BRIDGE_MODE_VEPA) && 13115 (mode != BRIDGE_MODE_VEB)) 13116 return -EINVAL; 13117 13118 /* Insert a new HW bridge */ 13119 if (!veb) { 13120 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 13121 vsi->tc_config.enabled_tc); 13122 if (veb) { 13123 veb->bridge_mode = mode; 13124 i40e_config_bridge_mode(veb); 13125 } else { 13126 /* No Bridge HW offload available */ 13127 return -ENOENT; 13128 } 13129 break; 13130 } else if (mode != veb->bridge_mode) { 13131 /* Existing HW bridge but different mode needs reset */ 13132 veb->bridge_mode = mode; 13133 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 13134 if (mode == BRIDGE_MODE_VEB) 13135 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13136 else 13137 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 13138 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 13139 break; 13140 } 13141 } 13142 13143 return 0; 13144} 13145 13146/** 13147 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 13148 * @skb: skb buff 13149 * @pid: process id 13150 * @seq: RTNL message seq # 13151 * @dev: the netdev being configured 13152 * @filter_mask: unused 13153 * @nlflags: netlink flags passed in 13154 * 13155 * Return the mode in which the hardware bridge is operating in 13156 * i.e VEB or VEPA. 13157 **/ 13158static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13159 struct net_device *dev, 13160 u32 __always_unused filter_mask, 13161 int nlflags) 13162{ 13163 struct i40e_netdev_priv *np = netdev_priv(dev); 13164 struct i40e_vsi *vsi = np->vsi; 13165 struct i40e_pf *pf = vsi->back; 13166 struct i40e_veb *veb; 13167 13168 /* Only for PF VSI for now */ 13169 if (vsi->type != I40E_VSI_MAIN) 13170 return -EOPNOTSUPP; 13171 13172 /* Find the HW bridge for the PF VSI */ 13173 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 13174 if (!veb) 13175 return 0; 13176 13177 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 13178 0, 0, nlflags, filter_mask, NULL); 13179} 13180 13181/** 13182 * i40e_features_check - Validate encapsulated packet conforms to limits 13183 * @skb: skb buff 13184 * @dev: This physical port's netdev 13185 * @features: Offload features that the stack believes apply 13186 **/ 13187static netdev_features_t i40e_features_check(struct sk_buff *skb, 13188 struct net_device *dev, 13189 netdev_features_t features) 13190{ 13191 size_t len; 13192 13193 /* No point in doing any of this if neither checksum nor GSO are 13194 * being requested for this frame. We can rule out both by just 13195 * checking for CHECKSUM_PARTIAL 13196 */ 13197 if (skb->ip_summed != CHECKSUM_PARTIAL) 13198 return features; 13199 13200 /* We cannot support GSO if the MSS is going to be less than 13201 * 64 bytes. If it is then we need to drop support for GSO. 13202 */ 13203 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 13204 features &= ~NETIF_F_GSO_MASK; 13205 13206 /* MACLEN can support at most 63 words */ 13207 len = skb_network_offset(skb); 13208 if (len & ~(63 * 2)) 13209 goto out_err; 13210 13211 /* IPLEN and EIPLEN can support at most 127 dwords */ 13212 len = skb_network_header_len(skb); 13213 if (len & ~(127 * 4)) 13214 goto out_err; 13215 13216 if (skb->encapsulation) { 13217 /* L4TUNLEN can support 127 words */ 13218 len = skb_inner_network_header(skb) - skb_transport_header(skb); 13219 if (len & ~(127 * 2)) 13220 goto out_err; 13221 13222 /* IPLEN can support at most 127 dwords */ 13223 len = skb_inner_transport_header(skb) - 13224 skb_inner_network_header(skb); 13225 if (len & ~(127 * 4)) 13226 goto out_err; 13227 } 13228 13229 /* No need to validate L4LEN as TCP is the only protocol with a 13230 * flexible value and we support all possible values supported 13231 * by TCP, which is at most 15 dwords 13232 */ 13233 13234 return features; 13235out_err: 13236 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 13237} 13238 13239/** 13240 * i40e_xdp_setup - add/remove an XDP program 13241 * @vsi: VSI to changed 13242 * @prog: XDP program 13243 * @extack: netlink extended ack 13244 **/ 13245static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog, 13246 struct netlink_ext_ack *extack) 13247{ 13248 int frame_size = i40e_max_vsi_frame_size(vsi, prog); 13249 struct i40e_pf *pf = vsi->back; 13250 struct bpf_prog *old_prog; 13251 bool need_reset; 13252 int i; 13253 13254 /* VSI shall be deleted in a moment, block loading new programs */ 13255 if (prog && test_bit(__I40E_IN_REMOVE, pf->state)) 13256 return -EINVAL; 13257 13258 /* Don't allow frames that span over multiple buffers */ 13259 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) { 13260 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags"); 13261 return -EINVAL; 13262 } 13263 13264 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 13265 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 13266 if (need_reset) 13267 i40e_prep_for_reset(pf); 13268 13269 old_prog = xchg(&vsi->xdp_prog, prog); 13270 13271 if (need_reset) { 13272 if (!prog) { 13273 xdp_features_clear_redirect_target(vsi->netdev); 13274 /* Wait until ndo_xsk_wakeup completes. */ 13275 synchronize_rcu(); 13276 } 13277 i40e_reset_and_rebuild(pf, true, true); 13278 } 13279 13280 if (!i40e_enabled_xdp_vsi(vsi) && prog) { 13281 if (i40e_realloc_rx_bi_zc(vsi, true)) 13282 return -ENOMEM; 13283 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) { 13284 if (i40e_realloc_rx_bi_zc(vsi, false)) 13285 return -ENOMEM; 13286 } 13287 13288 for (i = 0; i < vsi->num_queue_pairs; i++) 13289 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 13290 13291 if (old_prog) 13292 bpf_prog_put(old_prog); 13293 13294 /* Kick start the NAPI context if there is an AF_XDP socket open 13295 * on that queue id. This so that receiving will start. 13296 */ 13297 if (need_reset && prog) { 13298 for (i = 0; i < vsi->num_queue_pairs; i++) 13299 if (vsi->xdp_rings[i]->xsk_pool) 13300 (void)i40e_xsk_wakeup(vsi->netdev, i, 13301 XDP_WAKEUP_RX); 13302 xdp_features_set_redirect_target(vsi->netdev, true); 13303 } 13304 13305 return 0; 13306} 13307 13308/** 13309 * i40e_enter_busy_conf - Enters busy config state 13310 * @vsi: vsi 13311 * 13312 * Returns 0 on success, <0 for failure. 13313 **/ 13314static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 13315{ 13316 struct i40e_pf *pf = vsi->back; 13317 int timeout = 50; 13318 13319 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 13320 timeout--; 13321 if (!timeout) 13322 return -EBUSY; 13323 usleep_range(1000, 2000); 13324 } 13325 13326 return 0; 13327} 13328 13329/** 13330 * i40e_exit_busy_conf - Exits busy config state 13331 * @vsi: vsi 13332 **/ 13333static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 13334{ 13335 struct i40e_pf *pf = vsi->back; 13336 13337 clear_bit(__I40E_CONFIG_BUSY, pf->state); 13338} 13339 13340/** 13341 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 13342 * @vsi: vsi 13343 * @queue_pair: queue pair 13344 **/ 13345static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 13346{ 13347 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 13348 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 13349 memset(&vsi->tx_rings[queue_pair]->stats, 0, 13350 sizeof(vsi->tx_rings[queue_pair]->stats)); 13351 if (i40e_enabled_xdp_vsi(vsi)) { 13352 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 13353 sizeof(vsi->xdp_rings[queue_pair]->stats)); 13354 } 13355} 13356 13357/** 13358 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 13359 * @vsi: vsi 13360 * @queue_pair: queue pair 13361 **/ 13362static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 13363{ 13364 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 13365 if (i40e_enabled_xdp_vsi(vsi)) { 13366 /* Make sure that in-progress ndo_xdp_xmit calls are 13367 * completed. 13368 */ 13369 synchronize_rcu(); 13370 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 13371 } 13372 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13373} 13374 13375/** 13376 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 13377 * @vsi: vsi 13378 * @queue_pair: queue pair 13379 * @enable: true for enable, false for disable 13380 **/ 13381static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 13382 bool enable) 13383{ 13384 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13385 struct i40e_q_vector *q_vector = rxr->q_vector; 13386 13387 if (!vsi->netdev) 13388 return; 13389 13390 /* All rings in a qp belong to the same qvector. */ 13391 if (q_vector->rx.ring || q_vector->tx.ring) { 13392 if (enable) 13393 napi_enable(&q_vector->napi); 13394 else 13395 napi_disable(&q_vector->napi); 13396 } 13397} 13398 13399/** 13400 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 13401 * @vsi: vsi 13402 * @queue_pair: queue pair 13403 * @enable: true for enable, false for disable 13404 * 13405 * Returns 0 on success, <0 on failure. 13406 **/ 13407static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 13408 bool enable) 13409{ 13410 struct i40e_pf *pf = vsi->back; 13411 int pf_q, ret = 0; 13412 13413 pf_q = vsi->base_queue + queue_pair; 13414 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 13415 false /*is xdp*/, enable); 13416 if (ret) { 13417 dev_info(&pf->pdev->dev, 13418 "VSI seid %d Tx ring %d %sable timeout\n", 13419 vsi->seid, pf_q, (enable ? "en" : "dis")); 13420 return ret; 13421 } 13422 13423 i40e_control_rx_q(pf, pf_q, enable); 13424 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 13425 if (ret) { 13426 dev_info(&pf->pdev->dev, 13427 "VSI seid %d Rx ring %d %sable timeout\n", 13428 vsi->seid, pf_q, (enable ? "en" : "dis")); 13429 return ret; 13430 } 13431 13432 /* Due to HW errata, on Rx disable only, the register can 13433 * indicate done before it really is. Needs 50ms to be sure 13434 */ 13435 if (!enable) 13436 mdelay(50); 13437 13438 if (!i40e_enabled_xdp_vsi(vsi)) 13439 return ret; 13440 13441 ret = i40e_control_wait_tx_q(vsi->seid, pf, 13442 pf_q + vsi->alloc_queue_pairs, 13443 true /*is xdp*/, enable); 13444 if (ret) { 13445 dev_info(&pf->pdev->dev, 13446 "VSI seid %d XDP Tx ring %d %sable timeout\n", 13447 vsi->seid, pf_q, (enable ? "en" : "dis")); 13448 } 13449 13450 return ret; 13451} 13452 13453/** 13454 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 13455 * @vsi: vsi 13456 * @queue_pair: queue_pair 13457 **/ 13458static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 13459{ 13460 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13461 struct i40e_pf *pf = vsi->back; 13462 struct i40e_hw *hw = &pf->hw; 13463 13464 /* All rings in a qp belong to the same qvector. */ 13465 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 13466 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 13467 else 13468 i40e_irq_dynamic_enable_icr0(pf); 13469 13470 i40e_flush(hw); 13471} 13472 13473/** 13474 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 13475 * @vsi: vsi 13476 * @queue_pair: queue_pair 13477 **/ 13478static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 13479{ 13480 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13481 struct i40e_pf *pf = vsi->back; 13482 struct i40e_hw *hw = &pf->hw; 13483 13484 /* For simplicity, instead of removing the qp interrupt causes 13485 * from the interrupt linked list, we simply disable the interrupt, and 13486 * leave the list intact. 13487 * 13488 * All rings in a qp belong to the same qvector. 13489 */ 13490 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 13491 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 13492 13493 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 13494 i40e_flush(hw); 13495 synchronize_irq(pf->msix_entries[intpf].vector); 13496 } else { 13497 /* Legacy and MSI mode - this stops all interrupt handling */ 13498 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 13499 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 13500 i40e_flush(hw); 13501 synchronize_irq(pf->pdev->irq); 13502 } 13503} 13504 13505/** 13506 * i40e_queue_pair_disable - Disables a queue pair 13507 * @vsi: vsi 13508 * @queue_pair: queue pair 13509 * 13510 * Returns 0 on success, <0 on failure. 13511 **/ 13512int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 13513{ 13514 int err; 13515 13516 err = i40e_enter_busy_conf(vsi); 13517 if (err) 13518 return err; 13519 13520 i40e_queue_pair_disable_irq(vsi, queue_pair); 13521 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 13522 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 13523 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13524 i40e_queue_pair_clean_rings(vsi, queue_pair); 13525 i40e_queue_pair_reset_stats(vsi, queue_pair); 13526 13527 return err; 13528} 13529 13530/** 13531 * i40e_queue_pair_enable - Enables a queue pair 13532 * @vsi: vsi 13533 * @queue_pair: queue pair 13534 * 13535 * Returns 0 on success, <0 on failure. 13536 **/ 13537int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 13538{ 13539 int err; 13540 13541 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 13542 if (err) 13543 return err; 13544 13545 if (i40e_enabled_xdp_vsi(vsi)) { 13546 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 13547 if (err) 13548 return err; 13549 } 13550 13551 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 13552 if (err) 13553 return err; 13554 13555 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 13556 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 13557 i40e_queue_pair_enable_irq(vsi, queue_pair); 13558 13559 i40e_exit_busy_conf(vsi); 13560 13561 return err; 13562} 13563 13564/** 13565 * i40e_xdp - implements ndo_bpf for i40e 13566 * @dev: netdevice 13567 * @xdp: XDP command 13568 **/ 13569static int i40e_xdp(struct net_device *dev, 13570 struct netdev_bpf *xdp) 13571{ 13572 struct i40e_netdev_priv *np = netdev_priv(dev); 13573 struct i40e_vsi *vsi = np->vsi; 13574 13575 if (vsi->type != I40E_VSI_MAIN) 13576 return -EINVAL; 13577 13578 switch (xdp->command) { 13579 case XDP_SETUP_PROG: 13580 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack); 13581 case XDP_SETUP_XSK_POOL: 13582 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool, 13583 xdp->xsk.queue_id); 13584 default: 13585 return -EINVAL; 13586 } 13587} 13588 13589static const struct net_device_ops i40e_netdev_ops = { 13590 .ndo_open = i40e_open, 13591 .ndo_stop = i40e_close, 13592 .ndo_start_xmit = i40e_lan_xmit_frame, 13593 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 13594 .ndo_set_rx_mode = i40e_set_rx_mode, 13595 .ndo_validate_addr = eth_validate_addr, 13596 .ndo_set_mac_address = i40e_set_mac, 13597 .ndo_change_mtu = i40e_change_mtu, 13598 .ndo_tx_timeout = i40e_tx_timeout, 13599 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 13600 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 13601#ifdef CONFIG_NET_POLL_CONTROLLER 13602 .ndo_poll_controller = i40e_netpoll, 13603#endif 13604 .ndo_setup_tc = __i40e_setup_tc, 13605 .ndo_select_queue = i40e_lan_select_queue, 13606 .ndo_set_features = i40e_set_features, 13607 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 13608 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 13609 .ndo_get_vf_stats = i40e_get_vf_stats, 13610 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 13611 .ndo_get_vf_config = i40e_ndo_get_vf_config, 13612 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 13613 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 13614 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 13615 .ndo_get_phys_port_id = i40e_get_phys_port_id, 13616 .ndo_fdb_add = i40e_ndo_fdb_add, 13617 .ndo_features_check = i40e_features_check, 13618 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 13619 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 13620 .ndo_bpf = i40e_xdp, 13621 .ndo_xdp_xmit = i40e_xdp_xmit, 13622 .ndo_xsk_wakeup = i40e_xsk_wakeup, 13623 .ndo_dfwd_add_station = i40e_fwd_add, 13624 .ndo_dfwd_del_station = i40e_fwd_del, 13625 .ndo_hwtstamp_get = i40e_ptp_hwtstamp_get, 13626 .ndo_hwtstamp_set = i40e_ptp_hwtstamp_set, 13627}; 13628 13629/** 13630 * i40e_config_netdev - Setup the netdev flags 13631 * @vsi: the VSI being configured 13632 * 13633 * Returns 0 on success, negative value on failure 13634 **/ 13635static int i40e_config_netdev(struct i40e_vsi *vsi) 13636{ 13637 struct i40e_pf *pf = vsi->back; 13638 struct i40e_hw *hw = &pf->hw; 13639 struct i40e_netdev_priv *np; 13640 struct net_device *netdev; 13641 u8 broadcast[ETH_ALEN]; 13642 u8 mac_addr[ETH_ALEN]; 13643 int etherdev_size; 13644 netdev_features_t hw_enc_features; 13645 netdev_features_t hw_features; 13646 13647 etherdev_size = sizeof(struct i40e_netdev_priv); 13648 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 13649 if (!netdev) 13650 return -ENOMEM; 13651 13652 vsi->netdev = netdev; 13653 np = netdev_priv(netdev); 13654 np->vsi = vsi; 13655 13656 hw_enc_features = NETIF_F_SG | 13657 NETIF_F_HW_CSUM | 13658 NETIF_F_HIGHDMA | 13659 NETIF_F_SOFT_FEATURES | 13660 NETIF_F_TSO | 13661 NETIF_F_TSO_ECN | 13662 NETIF_F_TSO6 | 13663 NETIF_F_GSO_GRE | 13664 NETIF_F_GSO_GRE_CSUM | 13665 NETIF_F_GSO_PARTIAL | 13666 NETIF_F_GSO_IPXIP4 | 13667 NETIF_F_GSO_IPXIP6 | 13668 NETIF_F_GSO_UDP_TUNNEL | 13669 NETIF_F_GSO_UDP_TUNNEL_CSUM | 13670 NETIF_F_GSO_UDP_L4 | 13671 NETIF_F_SCTP_CRC | 13672 NETIF_F_RXHASH | 13673 NETIF_F_RXCSUM | 13674 0; 13675 13676 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps)) 13677 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 13678 13679 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; 13680 13681 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 13682 13683 netdev->hw_enc_features |= hw_enc_features; 13684 13685 /* record features VLANs can make use of */ 13686 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 13687 13688#define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 13689 NETIF_F_GSO_GRE_CSUM | \ 13690 NETIF_F_GSO_IPXIP4 | \ 13691 NETIF_F_GSO_IPXIP6 | \ 13692 NETIF_F_GSO_UDP_TUNNEL | \ 13693 NETIF_F_GSO_UDP_TUNNEL_CSUM) 13694 13695 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES; 13696 netdev->features |= NETIF_F_GSO_PARTIAL | 13697 I40E_GSO_PARTIAL_FEATURES; 13698 13699 netdev->mpls_features |= NETIF_F_SG; 13700 netdev->mpls_features |= NETIF_F_HW_CSUM; 13701 netdev->mpls_features |= NETIF_F_TSO; 13702 netdev->mpls_features |= NETIF_F_TSO6; 13703 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES; 13704 13705 /* enable macvlan offloads */ 13706 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 13707 13708 hw_features = hw_enc_features | 13709 NETIF_F_HW_VLAN_CTAG_TX | 13710 NETIF_F_HW_VLAN_CTAG_RX; 13711 13712 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags)) 13713 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 13714 13715 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK; 13716 13717 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 13718 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 13719 13720 netdev->features &= ~NETIF_F_HW_TC; 13721 13722 if (vsi->type == I40E_VSI_MAIN) { 13723 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 13724 ether_addr_copy(mac_addr, hw->mac.perm_addr); 13725 /* The following steps are necessary for two reasons. First, 13726 * some older NVM configurations load a default MAC-VLAN 13727 * filter that will accept any tagged packet, and we want to 13728 * replace this with a normal filter. Additionally, it is 13729 * possible our MAC address was provided by the platform using 13730 * Open Firmware or similar. 13731 * 13732 * Thus, we need to remove the default filter and install one 13733 * specific to the MAC address. 13734 */ 13735 i40e_rm_default_mac_filter(vsi, mac_addr); 13736 spin_lock_bh(&vsi->mac_filter_hash_lock); 13737 i40e_add_mac_filter(vsi, mac_addr); 13738 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13739 13740 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | 13741 NETDEV_XDP_ACT_REDIRECT | 13742 NETDEV_XDP_ACT_XSK_ZEROCOPY | 13743 NETDEV_XDP_ACT_RX_SG; 13744 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD; 13745 } else { 13746 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 13747 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 13748 * the end, which is 4 bytes long, so force truncation of the 13749 * original name by IFNAMSIZ - 4 13750 */ 13751 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 13752 13753 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4, 13754 main_vsi->netdev->name); 13755 eth_random_addr(mac_addr); 13756 13757 spin_lock_bh(&vsi->mac_filter_hash_lock); 13758 i40e_add_mac_filter(vsi, mac_addr); 13759 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13760 } 13761 13762 /* Add the broadcast filter so that we initially will receive 13763 * broadcast packets. Note that when a new VLAN is first added the 13764 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 13765 * specific filters as part of transitioning into "vlan" operation. 13766 * When more VLANs are added, the driver will copy each existing MAC 13767 * filter and add it for the new VLAN. 13768 * 13769 * Broadcast filters are handled specially by 13770 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13771 * promiscuous bit instead of adding this directly as a MAC/VLAN 13772 * filter. The subtask will update the correct broadcast promiscuous 13773 * bits as VLANs become active or inactive. 13774 */ 13775 eth_broadcast_addr(broadcast); 13776 spin_lock_bh(&vsi->mac_filter_hash_lock); 13777 i40e_add_mac_filter(vsi, broadcast); 13778 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13779 13780 eth_hw_addr_set(netdev, mac_addr); 13781 ether_addr_copy(netdev->perm_addr, mac_addr); 13782 13783 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13784 netdev->neigh_priv_len = sizeof(u32) * 4; 13785 13786 netdev->priv_flags |= IFF_UNICAST_FLT; 13787 netdev->priv_flags |= IFF_SUPP_NOFCS; 13788 /* Setup netdev TC information */ 13789 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13790 13791 netdev->netdev_ops = &i40e_netdev_ops; 13792 netdev->watchdog_timeo = 5 * HZ; 13793 i40e_set_ethtool_ops(netdev); 13794 13795 /* MTU range: 68 - 9706 */ 13796 netdev->min_mtu = ETH_MIN_MTU; 13797 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13798 13799 return 0; 13800} 13801 13802/** 13803 * i40e_vsi_delete - Delete a VSI from the switch 13804 * @vsi: the VSI being removed 13805 * 13806 * Returns 0 on success, negative value on failure 13807 **/ 13808static void i40e_vsi_delete(struct i40e_vsi *vsi) 13809{ 13810 /* remove default VSI is not allowed */ 13811 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13812 return; 13813 13814 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13815} 13816 13817/** 13818 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13819 * @vsi: the VSI being queried 13820 * 13821 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13822 **/ 13823int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13824{ 13825 struct i40e_veb *veb; 13826 struct i40e_pf *pf = vsi->back; 13827 13828 /* Uplink is not a bridge so default to VEB */ 13829 if (vsi->veb_idx >= I40E_MAX_VEB) 13830 return 1; 13831 13832 veb = pf->veb[vsi->veb_idx]; 13833 if (!veb) { 13834 dev_info(&pf->pdev->dev, 13835 "There is no veb associated with the bridge\n"); 13836 return -ENOENT; 13837 } 13838 13839 /* Uplink is a bridge in VEPA mode */ 13840 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13841 return 0; 13842 } else { 13843 /* Uplink is a bridge in VEB mode */ 13844 return 1; 13845 } 13846 13847 /* VEPA is now default bridge, so return 0 */ 13848 return 0; 13849} 13850 13851/** 13852 * i40e_add_vsi - Add a VSI to the switch 13853 * @vsi: the VSI being configured 13854 * 13855 * This initializes a VSI context depending on the VSI type to be added and 13856 * passes it down to the add_vsi aq command. 13857 **/ 13858static int i40e_add_vsi(struct i40e_vsi *vsi) 13859{ 13860 int ret = -ENODEV; 13861 struct i40e_pf *pf = vsi->back; 13862 struct i40e_hw *hw = &pf->hw; 13863 struct i40e_vsi_context ctxt; 13864 struct i40e_mac_filter *f; 13865 struct hlist_node *h; 13866 int bkt; 13867 13868 u8 enabled_tc = 0x1; /* TC0 enabled */ 13869 int f_count = 0; 13870 13871 memset(&ctxt, 0, sizeof(ctxt)); 13872 switch (vsi->type) { 13873 case I40E_VSI_MAIN: 13874 /* The PF's main VSI is already setup as part of the 13875 * device initialization, so we'll not bother with 13876 * the add_vsi call, but we will retrieve the current 13877 * VSI context. 13878 */ 13879 ctxt.seid = pf->main_vsi_seid; 13880 ctxt.pf_num = pf->hw.pf_id; 13881 ctxt.vf_num = 0; 13882 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13883 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13884 if (ret) { 13885 dev_info(&pf->pdev->dev, 13886 "couldn't get PF vsi config, err %pe aq_err %s\n", 13887 ERR_PTR(ret), 13888 libie_aq_str(pf->hw.aq.asq_last_status)); 13889 return -ENOENT; 13890 } 13891 vsi->info = ctxt.info; 13892 vsi->info.valid_sections = 0; 13893 13894 vsi->seid = ctxt.seid; 13895 vsi->id = ctxt.vsi_number; 13896 13897 enabled_tc = i40e_pf_get_tc_map(pf); 13898 13899 /* Source pruning is enabled by default, so the flag is 13900 * negative logic - if it's set, we need to fiddle with 13901 * the VSI to disable source pruning. 13902 */ 13903 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) { 13904 memset(&ctxt, 0, sizeof(ctxt)); 13905 ctxt.seid = pf->main_vsi_seid; 13906 ctxt.pf_num = pf->hw.pf_id; 13907 ctxt.vf_num = 0; 13908 ctxt.info.valid_sections |= 13909 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13910 ctxt.info.switch_id = 13911 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13912 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13913 if (ret) { 13914 dev_info(&pf->pdev->dev, 13915 "update vsi failed, err %d aq_err %s\n", 13916 ret, 13917 libie_aq_str(pf->hw.aq.asq_last_status)); 13918 ret = -ENOENT; 13919 goto err; 13920 } 13921 } 13922 13923 /* MFP mode setup queue map and update VSI */ 13924 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && 13925 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13926 memset(&ctxt, 0, sizeof(ctxt)); 13927 ctxt.seid = pf->main_vsi_seid; 13928 ctxt.pf_num = pf->hw.pf_id; 13929 ctxt.vf_num = 0; 13930 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13931 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13932 if (ret) { 13933 dev_info(&pf->pdev->dev, 13934 "update vsi failed, err %pe aq_err %s\n", 13935 ERR_PTR(ret), 13936 libie_aq_str(pf->hw.aq.asq_last_status)); 13937 ret = -ENOENT; 13938 goto err; 13939 } 13940 /* update the local VSI info queue map */ 13941 i40e_vsi_update_queue_map(vsi, &ctxt); 13942 vsi->info.valid_sections = 0; 13943 } else { 13944 /* Default/Main VSI is only enabled for TC0 13945 * reconfigure it to enable all TCs that are 13946 * available on the port in SFP mode. 13947 * For MFP case the iSCSI PF would use this 13948 * flow to enable LAN+iSCSI TC. 13949 */ 13950 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13951 if (ret) { 13952 /* Single TC condition is not fatal, 13953 * message and continue 13954 */ 13955 dev_info(&pf->pdev->dev, 13956 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n", 13957 enabled_tc, 13958 ERR_PTR(ret), 13959 libie_aq_str(pf->hw.aq.asq_last_status)); 13960 } 13961 } 13962 break; 13963 13964 case I40E_VSI_FDIR: 13965 ctxt.pf_num = hw->pf_id; 13966 ctxt.vf_num = 0; 13967 ctxt.uplink_seid = vsi->uplink_seid; 13968 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13969 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13970 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) && 13971 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13972 ctxt.info.valid_sections |= 13973 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13974 ctxt.info.switch_id = 13975 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13976 } 13977 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13978 break; 13979 13980 case I40E_VSI_VMDQ2: 13981 ctxt.pf_num = hw->pf_id; 13982 ctxt.vf_num = 0; 13983 ctxt.uplink_seid = vsi->uplink_seid; 13984 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13985 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13986 13987 /* This VSI is connected to VEB so the switch_id 13988 * should be set to zero by default. 13989 */ 13990 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13991 ctxt.info.valid_sections |= 13992 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13993 ctxt.info.switch_id = 13994 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13995 } 13996 13997 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13998 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13999 break; 14000 14001 case I40E_VSI_SRIOV: 14002 ctxt.pf_num = hw->pf_id; 14003 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 14004 ctxt.uplink_seid = vsi->uplink_seid; 14005 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14006 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 14007 14008 /* This VSI is connected to VEB so the switch_id 14009 * should be set to zero by default. 14010 */ 14011 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14012 ctxt.info.valid_sections |= 14013 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14014 ctxt.info.switch_id = 14015 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14016 } 14017 14018 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) { 14019 ctxt.info.valid_sections |= 14020 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 14021 ctxt.info.queueing_opt_flags |= 14022 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 14023 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 14024 } 14025 14026 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 14027 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 14028 if (pf->vf[vsi->vf_id].spoofchk) { 14029 ctxt.info.valid_sections |= 14030 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 14031 ctxt.info.sec_flags |= 14032 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 14033 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 14034 } 14035 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14036 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14037 break; 14038 14039 case I40E_VSI_IWARP: 14040 /* send down message to iWARP */ 14041 break; 14042 14043 default: 14044 return -ENODEV; 14045 } 14046 14047 if (vsi->type != I40E_VSI_MAIN) { 14048 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 14049 if (ret) { 14050 dev_info(&vsi->back->pdev->dev, 14051 "add vsi failed, err %pe aq_err %s\n", 14052 ERR_PTR(ret), 14053 libie_aq_str(pf->hw.aq.asq_last_status)); 14054 ret = -ENOENT; 14055 goto err; 14056 } 14057 vsi->info = ctxt.info; 14058 vsi->info.valid_sections = 0; 14059 vsi->seid = ctxt.seid; 14060 vsi->id = ctxt.vsi_number; 14061 } 14062 14063 spin_lock_bh(&vsi->mac_filter_hash_lock); 14064 vsi->active_filters = 0; 14065 /* If macvlan filters already exist, force them to get loaded */ 14066 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 14067 f->state = I40E_FILTER_NEW; 14068 f_count++; 14069 } 14070 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14071 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 14072 14073 if (f_count) { 14074 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 14075 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 14076 } 14077 14078 /* Update VSI BW information */ 14079 ret = i40e_vsi_get_bw_info(vsi); 14080 if (ret) { 14081 dev_info(&pf->pdev->dev, 14082 "couldn't get vsi bw info, err %pe aq_err %s\n", 14083 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14084 /* VSI is already added so not tearing that up */ 14085 ret = 0; 14086 } 14087 14088err: 14089 return ret; 14090} 14091 14092/** 14093 * i40e_vsi_release - Delete a VSI and free its resources 14094 * @vsi: the VSI being removed 14095 * 14096 * Returns 0 on success or < 0 on error 14097 **/ 14098int i40e_vsi_release(struct i40e_vsi *vsi) 14099{ 14100 struct i40e_mac_filter *f; 14101 struct hlist_node *h; 14102 struct i40e_veb *veb; 14103 struct i40e_pf *pf; 14104 u16 uplink_seid; 14105 int i, n, bkt; 14106 14107 pf = vsi->back; 14108 14109 /* release of a VEB-owner or last VSI is not allowed */ 14110 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 14111 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 14112 vsi->seid, vsi->uplink_seid); 14113 return -ENODEV; 14114 } 14115 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) { 14116 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 14117 return -ENODEV; 14118 } 14119 set_bit(__I40E_VSI_RELEASING, vsi->state); 14120 uplink_seid = vsi->uplink_seid; 14121 14122 if (vsi->type != I40E_VSI_SRIOV) { 14123 if (vsi->netdev_registered) { 14124 vsi->netdev_registered = false; 14125 if (vsi->netdev) { 14126 /* results in a call to i40e_close() */ 14127 unregister_netdev(vsi->netdev); 14128 } 14129 } else { 14130 i40e_vsi_close(vsi); 14131 } 14132 i40e_vsi_disable_irq(vsi); 14133 } 14134 14135 if (vsi->type == I40E_VSI_MAIN) 14136 i40e_devlink_destroy_port(pf); 14137 14138 spin_lock_bh(&vsi->mac_filter_hash_lock); 14139 14140 /* clear the sync flag on all filters */ 14141 if (vsi->netdev) { 14142 __dev_uc_unsync(vsi->netdev, NULL); 14143 __dev_mc_unsync(vsi->netdev, NULL); 14144 } 14145 14146 /* make sure any remaining filters are marked for deletion */ 14147 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 14148 __i40e_del_filter(vsi, f); 14149 14150 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14151 14152 i40e_sync_vsi_filters(vsi); 14153 14154 i40e_vsi_delete(vsi); 14155 i40e_vsi_free_q_vectors(vsi); 14156 if (vsi->netdev) { 14157 free_netdev(vsi->netdev); 14158 vsi->netdev = NULL; 14159 } 14160 i40e_vsi_clear_rings(vsi); 14161 i40e_vsi_clear(vsi); 14162 14163 /* If this was the last thing on the VEB, except for the 14164 * controlling VSI, remove the VEB, which puts the controlling 14165 * VSI onto the uplink port. 14166 * 14167 * Well, okay, there's one more exception here: don't remove 14168 * the floating VEBs yet. We'll wait for an explicit remove request 14169 * from up the network stack. 14170 */ 14171 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14172 if (veb && veb->uplink_seid) { 14173 n = 0; 14174 14175 /* Count non-controlling VSIs present on the VEB */ 14176 i40e_pf_for_each_vsi(pf, i, vsi) 14177 if (vsi->uplink_seid == uplink_seid && 14178 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14179 n++; 14180 14181 /* If there is no VSI except the control one then release 14182 * the VEB and put the control VSI onto VEB uplink. 14183 */ 14184 if (!n) 14185 i40e_veb_release(veb); 14186 } 14187 14188 return 0; 14189} 14190 14191/** 14192 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 14193 * @vsi: ptr to the VSI 14194 * 14195 * This should only be called after i40e_vsi_mem_alloc() which allocates the 14196 * corresponding SW VSI structure and initializes num_queue_pairs for the 14197 * newly allocated VSI. 14198 * 14199 * Returns 0 on success or negative on failure 14200 **/ 14201static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 14202{ 14203 int ret = -ENOENT; 14204 struct i40e_pf *pf = vsi->back; 14205 14206 if (vsi->q_vectors[0]) { 14207 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 14208 vsi->seid); 14209 return -EEXIST; 14210 } 14211 14212 if (vsi->base_vector) { 14213 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 14214 vsi->seid, vsi->base_vector); 14215 return -EEXIST; 14216 } 14217 14218 ret = i40e_vsi_alloc_q_vectors(vsi); 14219 if (ret) { 14220 dev_info(&pf->pdev->dev, 14221 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 14222 vsi->num_q_vectors, vsi->seid, ret); 14223 vsi->num_q_vectors = 0; 14224 goto vector_setup_out; 14225 } 14226 14227 /* In Legacy mode, we do not have to get any other vector since we 14228 * piggyback on the misc/ICR0 for queue interrupts. 14229 */ 14230 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 14231 return ret; 14232 if (vsi->num_q_vectors) 14233 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 14234 vsi->num_q_vectors, vsi->idx); 14235 if (vsi->base_vector < 0) { 14236 dev_info(&pf->pdev->dev, 14237 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 14238 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 14239 i40e_vsi_free_q_vectors(vsi); 14240 ret = -ENOENT; 14241 goto vector_setup_out; 14242 } 14243 14244vector_setup_out: 14245 return ret; 14246} 14247 14248/** 14249 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 14250 * @vsi: pointer to the vsi. 14251 * 14252 * This re-allocates a vsi's queue resources. 14253 * 14254 * Returns pointer to the successfully allocated and configured VSI sw struct 14255 * on success, otherwise returns NULL on failure. 14256 **/ 14257static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 14258{ 14259 struct i40e_vsi *main_vsi; 14260 u16 alloc_queue_pairs; 14261 struct i40e_pf *pf; 14262 int ret; 14263 14264 if (!vsi) 14265 return NULL; 14266 14267 pf = vsi->back; 14268 14269 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 14270 i40e_vsi_clear_rings(vsi); 14271 14272 i40e_vsi_free_arrays(vsi, false); 14273 i40e_set_num_rings_in_vsi(vsi); 14274 ret = i40e_vsi_alloc_arrays(vsi, false); 14275 if (ret) 14276 goto err_vsi; 14277 14278 alloc_queue_pairs = vsi->alloc_queue_pairs * 14279 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14280 14281 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14282 if (ret < 0) { 14283 dev_info(&pf->pdev->dev, 14284 "failed to get tracking for %d queues for VSI %d err %d\n", 14285 alloc_queue_pairs, vsi->seid, ret); 14286 goto err_vsi; 14287 } 14288 vsi->base_queue = ret; 14289 14290 /* Update the FW view of the VSI. Force a reset of TC and queue 14291 * layout configurations. 14292 */ 14293 main_vsi = i40e_pf_get_main_vsi(pf); 14294 main_vsi->seid = pf->main_vsi_seid; 14295 i40e_vsi_reconfig_tc(main_vsi); 14296 14297 if (vsi->type == I40E_VSI_MAIN) 14298 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 14299 14300 /* assign it some queues */ 14301 ret = i40e_alloc_rings(vsi); 14302 if (ret) 14303 goto err_rings; 14304 14305 /* map all of the rings to the q_vectors */ 14306 i40e_vsi_map_rings_to_vectors(vsi); 14307 return vsi; 14308 14309err_rings: 14310 i40e_vsi_free_q_vectors(vsi); 14311 if (vsi->netdev_registered) { 14312 vsi->netdev_registered = false; 14313 unregister_netdev(vsi->netdev); 14314 free_netdev(vsi->netdev); 14315 vsi->netdev = NULL; 14316 } 14317 if (vsi->type == I40E_VSI_MAIN) 14318 i40e_devlink_destroy_port(pf); 14319 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14320err_vsi: 14321 i40e_vsi_clear(vsi); 14322 return NULL; 14323} 14324 14325/** 14326 * i40e_vsi_setup - Set up a VSI by a given type 14327 * @pf: board private structure 14328 * @type: VSI type 14329 * @uplink_seid: the switch element to link to 14330 * @param1: usage depends upon VSI type. For VF types, indicates VF id 14331 * 14332 * This allocates the sw VSI structure and its queue resources, then add a VSI 14333 * to the identified VEB. 14334 * 14335 * Returns pointer to the successfully allocated and configure VSI sw struct on 14336 * success, otherwise returns NULL on failure. 14337 **/ 14338struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 14339 u16 uplink_seid, u32 param1) 14340{ 14341 struct i40e_vsi *vsi = NULL; 14342 struct i40e_veb *veb = NULL; 14343 u16 alloc_queue_pairs; 14344 int v_idx; 14345 int ret; 14346 14347 /* The requested uplink_seid must be either 14348 * - the PF's port seid 14349 * no VEB is needed because this is the PF 14350 * or this is a Flow Director special case VSI 14351 * - seid of an existing VEB 14352 * - seid of a VSI that owns an existing VEB 14353 * - seid of a VSI that doesn't own a VEB 14354 * a new VEB is created and the VSI becomes the owner 14355 * - seid of the PF VSI, which is what creates the first VEB 14356 * this is a special case of the previous 14357 * 14358 * Find which uplink_seid we were given and create a new VEB if needed 14359 */ 14360 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid); 14361 if (!veb && uplink_seid != pf->mac_seid) { 14362 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid); 14363 if (!vsi) { 14364 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 14365 uplink_seid); 14366 return NULL; 14367 } 14368 14369 if (vsi->uplink_seid == pf->mac_seid) 14370 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid, 14371 vsi->tc_config.enabled_tc); 14372 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14373 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid, 14374 vsi->tc_config.enabled_tc); 14375 if (veb) { 14376 if (vsi->type != I40E_VSI_MAIN) { 14377 dev_info(&vsi->back->pdev->dev, 14378 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 14379 return NULL; 14380 } 14381 /* We come up by default in VEPA mode if SRIOV is not 14382 * already enabled, in which case we can't force VEPA 14383 * mode. 14384 */ 14385 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) { 14386 veb->bridge_mode = BRIDGE_MODE_VEPA; 14387 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 14388 } 14389 i40e_config_bridge_mode(veb); 14390 } 14391 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid); 14392 if (!veb) { 14393 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 14394 return NULL; 14395 } 14396 14397 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14398 uplink_seid = veb->seid; 14399 } 14400 14401 /* get vsi sw struct */ 14402 v_idx = i40e_vsi_mem_alloc(pf, type); 14403 if (v_idx < 0) 14404 goto err_alloc; 14405 vsi = pf->vsi[v_idx]; 14406 if (!vsi) 14407 goto err_alloc; 14408 vsi->type = type; 14409 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 14410 14411 if (type == I40E_VSI_MAIN) 14412 pf->lan_vsi = v_idx; 14413 else if (type == I40E_VSI_SRIOV) 14414 vsi->vf_id = param1; 14415 /* assign it some queues */ 14416 alloc_queue_pairs = vsi->alloc_queue_pairs * 14417 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14418 14419 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14420 if (ret < 0) { 14421 dev_info(&pf->pdev->dev, 14422 "failed to get tracking for %d queues for VSI %d err=%d\n", 14423 alloc_queue_pairs, vsi->seid, ret); 14424 goto err_vsi; 14425 } 14426 vsi->base_queue = ret; 14427 14428 /* get a VSI from the hardware */ 14429 vsi->uplink_seid = uplink_seid; 14430 ret = i40e_add_vsi(vsi); 14431 if (ret) 14432 goto err_vsi; 14433 14434 switch (vsi->type) { 14435 /* setup the netdev if needed */ 14436 case I40E_VSI_MAIN: 14437 case I40E_VSI_VMDQ2: 14438 ret = i40e_config_netdev(vsi); 14439 if (ret) 14440 goto err_netdev; 14441 ret = i40e_netif_set_realnum_tx_rx_queues(vsi); 14442 if (ret) 14443 goto err_netdev; 14444 if (vsi->type == I40E_VSI_MAIN) { 14445 ret = i40e_devlink_create_port(pf); 14446 if (ret) 14447 goto err_netdev; 14448 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port); 14449 } 14450 ret = register_netdev(vsi->netdev); 14451 if (ret) 14452 goto err_dl_port; 14453 vsi->netdev_registered = true; 14454 netif_carrier_off(vsi->netdev); 14455#ifdef CONFIG_I40E_DCB 14456 /* Setup DCB netlink interface */ 14457 i40e_dcbnl_setup(vsi); 14458#endif /* CONFIG_I40E_DCB */ 14459 fallthrough; 14460 case I40E_VSI_FDIR: 14461 /* set up vectors and rings if needed */ 14462 ret = i40e_vsi_setup_vectors(vsi); 14463 if (ret) 14464 goto err_msix; 14465 14466 ret = i40e_alloc_rings(vsi); 14467 if (ret) 14468 goto err_rings; 14469 14470 /* map all of the rings to the q_vectors */ 14471 i40e_vsi_map_rings_to_vectors(vsi); 14472 14473 i40e_vsi_reset_stats(vsi); 14474 break; 14475 default: 14476 /* no netdev or rings for the other VSI types */ 14477 break; 14478 } 14479 14480 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) && 14481 vsi->type == I40E_VSI_VMDQ2) { 14482 ret = i40e_vsi_config_rss(vsi); 14483 if (ret) 14484 goto err_config; 14485 } 14486 return vsi; 14487 14488err_config: 14489 i40e_vsi_clear_rings(vsi); 14490err_rings: 14491 i40e_vsi_free_q_vectors(vsi); 14492err_msix: 14493 if (vsi->netdev_registered) { 14494 vsi->netdev_registered = false; 14495 unregister_netdev(vsi->netdev); 14496 free_netdev(vsi->netdev); 14497 vsi->netdev = NULL; 14498 } 14499err_dl_port: 14500 if (vsi->type == I40E_VSI_MAIN) 14501 i40e_devlink_destroy_port(pf); 14502err_netdev: 14503 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14504err_vsi: 14505 i40e_vsi_clear(vsi); 14506err_alloc: 14507 return NULL; 14508} 14509 14510/** 14511 * i40e_veb_get_bw_info - Query VEB BW information 14512 * @veb: the veb to query 14513 * 14514 * Query the Tx scheduler BW configuration data for given VEB 14515 **/ 14516static int i40e_veb_get_bw_info(struct i40e_veb *veb) 14517{ 14518 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 14519 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 14520 struct i40e_pf *pf = veb->pf; 14521 struct i40e_hw *hw = &pf->hw; 14522 u32 tc_bw_max; 14523 int ret = 0; 14524 int i; 14525 14526 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 14527 &bw_data, NULL); 14528 if (ret) { 14529 dev_info(&pf->pdev->dev, 14530 "query veb bw config failed, err %pe aq_err %s\n", 14531 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 14532 goto out; 14533 } 14534 14535 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 14536 &ets_data, NULL); 14537 if (ret) { 14538 dev_info(&pf->pdev->dev, 14539 "query veb bw ets config failed, err %pe aq_err %s\n", 14540 ERR_PTR(ret), libie_aq_str(hw->aq.asq_last_status)); 14541 goto out; 14542 } 14543 14544 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 14545 veb->bw_max_quanta = ets_data.tc_bw_max; 14546 veb->is_abs_credits = bw_data.absolute_credits_enable; 14547 veb->enabled_tc = ets_data.tc_valid_bits; 14548 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 14549 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 14550 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 14551 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 14552 veb->bw_tc_limit_credits[i] = 14553 le16_to_cpu(bw_data.tc_bw_limits[i]); 14554 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 14555 } 14556 14557out: 14558 return ret; 14559} 14560 14561/** 14562 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 14563 * @pf: board private structure 14564 * 14565 * On error: returns error code (negative) 14566 * On success: returns vsi index in PF (positive) 14567 **/ 14568static int i40e_veb_mem_alloc(struct i40e_pf *pf) 14569{ 14570 int ret = -ENOENT; 14571 struct i40e_veb *veb; 14572 int i; 14573 14574 /* Need to protect the allocation of switch elements at the PF level */ 14575 mutex_lock(&pf->switch_mutex); 14576 14577 /* VEB list may be fragmented if VEB creation/destruction has 14578 * been happening. We can afford to do a quick scan to look 14579 * for any free slots in the list. 14580 * 14581 * find next empty veb slot, looping back around if necessary 14582 */ 14583 i = 0; 14584 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 14585 i++; 14586 if (i >= I40E_MAX_VEB) { 14587 ret = -ENOMEM; 14588 goto err_alloc_veb; /* out of VEB slots! */ 14589 } 14590 14591 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 14592 if (!veb) { 14593 ret = -ENOMEM; 14594 goto err_alloc_veb; 14595 } 14596 veb->pf = pf; 14597 veb->idx = i; 14598 veb->enabled_tc = 1; 14599 14600 pf->veb[i] = veb; 14601 ret = i; 14602err_alloc_veb: 14603 mutex_unlock(&pf->switch_mutex); 14604 return ret; 14605} 14606 14607/** 14608 * i40e_switch_branch_release - Delete a branch of the switch tree 14609 * @branch: where to start deleting 14610 * 14611 * This uses recursion to find the tips of the branch to be 14612 * removed, deleting until we get back to and can delete this VEB. 14613 **/ 14614static void i40e_switch_branch_release(struct i40e_veb *branch) 14615{ 14616 struct i40e_pf *pf = branch->pf; 14617 u16 branch_seid = branch->seid; 14618 u16 veb_idx = branch->idx; 14619 struct i40e_vsi *vsi; 14620 struct i40e_veb *veb; 14621 int i; 14622 14623 /* release any VEBs on this VEB - RECURSION */ 14624 i40e_pf_for_each_veb(pf, i, veb) 14625 if (veb->uplink_seid == branch->seid) 14626 i40e_switch_branch_release(veb); 14627 14628 /* Release the VSIs on this VEB, but not the owner VSI. 14629 * 14630 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 14631 * the VEB itself, so don't use (*branch) after this loop. 14632 */ 14633 i40e_pf_for_each_vsi(pf, i, vsi) 14634 if (vsi->uplink_seid == branch_seid && 14635 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14636 i40e_vsi_release(vsi); 14637 14638 /* There's one corner case where the VEB might not have been 14639 * removed, so double check it here and remove it if needed. 14640 * This case happens if the veb was created from the debugfs 14641 * commands and no VSIs were added to it. 14642 */ 14643 if (pf->veb[veb_idx]) 14644 i40e_veb_release(pf->veb[veb_idx]); 14645} 14646 14647/** 14648 * i40e_veb_clear - remove veb struct 14649 * @veb: the veb to remove 14650 **/ 14651static void i40e_veb_clear(struct i40e_veb *veb) 14652{ 14653 if (!veb) 14654 return; 14655 14656 if (veb->pf) { 14657 struct i40e_pf *pf = veb->pf; 14658 14659 mutex_lock(&pf->switch_mutex); 14660 if (pf->veb[veb->idx] == veb) 14661 pf->veb[veb->idx] = NULL; 14662 mutex_unlock(&pf->switch_mutex); 14663 } 14664 14665 kfree(veb); 14666} 14667 14668/** 14669 * i40e_veb_release - Delete a VEB and free its resources 14670 * @veb: the VEB being removed 14671 **/ 14672void i40e_veb_release(struct i40e_veb *veb) 14673{ 14674 struct i40e_vsi *vsi, *vsi_it; 14675 struct i40e_pf *pf; 14676 int i, n = 0; 14677 14678 pf = veb->pf; 14679 14680 /* find the remaining VSI and check for extras */ 14681 i40e_pf_for_each_vsi(pf, i, vsi_it) 14682 if (vsi_it->uplink_seid == veb->seid) { 14683 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER) 14684 vsi = vsi_it; 14685 n++; 14686 } 14687 14688 /* Floating VEB has to be empty and regular one must have 14689 * single owner VSI. 14690 */ 14691 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) { 14692 dev_info(&pf->pdev->dev, 14693 "can't remove VEB %d with %d VSIs left\n", 14694 veb->seid, n); 14695 return; 14696 } 14697 14698 /* For regular VEB move the owner VSI to uplink port */ 14699 if (veb->uplink_seid) { 14700 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 14701 vsi->uplink_seid = veb->uplink_seid; 14702 vsi->veb_idx = I40E_NO_VEB; 14703 } 14704 14705 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14706 i40e_veb_clear(veb); 14707} 14708 14709/** 14710 * i40e_add_veb - create the VEB in the switch 14711 * @veb: the VEB to be instantiated 14712 * @vsi: the controlling VSI 14713 **/ 14714static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 14715{ 14716 struct i40e_pf *pf = veb->pf; 14717 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags); 14718 int ret; 14719 14720 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0, 14721 veb->enabled_tc, vsi ? false : true, 14722 &veb->seid, enable_stats, NULL); 14723 14724 /* get a VEB from the hardware */ 14725 if (ret) { 14726 dev_info(&pf->pdev->dev, 14727 "couldn't add VEB, err %pe aq_err %s\n", 14728 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14729 return -EPERM; 14730 } 14731 14732 /* get statistics counter */ 14733 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 14734 &veb->stats_idx, NULL, NULL, NULL); 14735 if (ret) { 14736 dev_info(&pf->pdev->dev, 14737 "couldn't get VEB statistics idx, err %pe aq_err %s\n", 14738 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14739 return -EPERM; 14740 } 14741 ret = i40e_veb_get_bw_info(veb); 14742 if (ret) { 14743 dev_info(&pf->pdev->dev, 14744 "couldn't get VEB bw info, err %pe aq_err %s\n", 14745 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14746 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14747 return -ENOENT; 14748 } 14749 14750 if (vsi) { 14751 vsi->uplink_seid = veb->seid; 14752 vsi->veb_idx = veb->idx; 14753 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14754 } 14755 14756 return 0; 14757} 14758 14759/** 14760 * i40e_veb_setup - Set up a VEB 14761 * @pf: board private structure 14762 * @uplink_seid: the switch element to link to 14763 * @vsi_seid: the initial VSI seid 14764 * @enabled_tc: Enabled TC bit-map 14765 * 14766 * This allocates the sw VEB structure and links it into the switch 14767 * It is possible and legal for this to be a duplicate of an already 14768 * existing VEB. It is also possible for both uplink and vsi seids 14769 * to be zero, in order to create a floating VEB. 14770 * 14771 * Returns pointer to the successfully allocated VEB sw struct on 14772 * success, otherwise returns NULL on failure. 14773 **/ 14774struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid, 14775 u16 vsi_seid, u8 enabled_tc) 14776{ 14777 struct i40e_vsi *vsi = NULL; 14778 struct i40e_veb *veb; 14779 int veb_idx; 14780 int ret; 14781 14782 /* if one seid is 0, the other must be 0 to create a floating relay */ 14783 if ((uplink_seid == 0 || vsi_seid == 0) && 14784 (uplink_seid + vsi_seid != 0)) { 14785 dev_info(&pf->pdev->dev, 14786 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14787 uplink_seid, vsi_seid); 14788 return NULL; 14789 } 14790 14791 /* make sure there is such a vsi and uplink */ 14792 if (vsi_seid) { 14793 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid); 14794 if (!vsi) { 14795 dev_err(&pf->pdev->dev, "vsi seid %d not found\n", 14796 vsi_seid); 14797 return NULL; 14798 } 14799 } 14800 14801 /* get veb sw struct */ 14802 veb_idx = i40e_veb_mem_alloc(pf); 14803 if (veb_idx < 0) 14804 goto err_alloc; 14805 veb = pf->veb[veb_idx]; 14806 veb->uplink_seid = uplink_seid; 14807 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14808 14809 /* create the VEB in the switch */ 14810 ret = i40e_add_veb(veb, vsi); 14811 if (ret) 14812 goto err_veb; 14813 14814 if (vsi && vsi->idx == pf->lan_vsi) 14815 pf->lan_veb = veb->idx; 14816 14817 return veb; 14818 14819err_veb: 14820 i40e_veb_clear(veb); 14821err_alloc: 14822 return NULL; 14823} 14824 14825/** 14826 * i40e_setup_pf_switch_element - set PF vars based on switch type 14827 * @pf: board private structure 14828 * @ele: element we are building info from 14829 * @num_reported: total number of elements 14830 * @printconfig: should we print the contents 14831 * 14832 * helper function to assist in extracting a few useful SEID values. 14833 **/ 14834static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14835 struct i40e_aqc_switch_config_element_resp *ele, 14836 u16 num_reported, bool printconfig) 14837{ 14838 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14839 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14840 u8 element_type = ele->element_type; 14841 u16 seid = le16_to_cpu(ele->seid); 14842 struct i40e_veb *veb; 14843 14844 if (printconfig) 14845 dev_info(&pf->pdev->dev, 14846 "type=%d seid=%d uplink=%d downlink=%d\n", 14847 element_type, seid, uplink_seid, downlink_seid); 14848 14849 switch (element_type) { 14850 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14851 pf->mac_seid = seid; 14852 break; 14853 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14854 /* Main VEB? */ 14855 if (uplink_seid != pf->mac_seid) 14856 break; 14857 veb = i40e_pf_get_main_veb(pf); 14858 if (!veb) { 14859 int v; 14860 14861 /* find existing or else empty VEB */ 14862 veb = i40e_pf_get_veb_by_seid(pf, seid); 14863 if (veb) { 14864 pf->lan_veb = veb->idx; 14865 } else { 14866 v = i40e_veb_mem_alloc(pf); 14867 if (v < 0) 14868 break; 14869 pf->lan_veb = v; 14870 } 14871 } 14872 14873 /* Try to get again main VEB as pf->lan_veb may have changed */ 14874 veb = i40e_pf_get_main_veb(pf); 14875 if (!veb) 14876 break; 14877 14878 veb->seid = seid; 14879 veb->uplink_seid = pf->mac_seid; 14880 veb->pf = pf; 14881 break; 14882 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14883 if (num_reported != 1) 14884 break; 14885 /* This is immediately after a reset so we can assume this is 14886 * the PF's VSI 14887 */ 14888 pf->mac_seid = uplink_seid; 14889 pf->main_vsi_seid = seid; 14890 if (printconfig) 14891 dev_info(&pf->pdev->dev, 14892 "pf_seid=%d main_vsi_seid=%d\n", 14893 downlink_seid, pf->main_vsi_seid); 14894 break; 14895 case I40E_SWITCH_ELEMENT_TYPE_PF: 14896 case I40E_SWITCH_ELEMENT_TYPE_VF: 14897 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14898 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14899 case I40E_SWITCH_ELEMENT_TYPE_PE: 14900 case I40E_SWITCH_ELEMENT_TYPE_PA: 14901 /* ignore these for now */ 14902 break; 14903 default: 14904 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14905 element_type, seid); 14906 break; 14907 } 14908} 14909 14910/** 14911 * i40e_fetch_switch_configuration - Get switch config from firmware 14912 * @pf: board private structure 14913 * @printconfig: should we print the contents 14914 * 14915 * Get the current switch configuration from the device and 14916 * extract a few useful SEID values. 14917 **/ 14918int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14919{ 14920 struct i40e_aqc_get_switch_config_resp *sw_config; 14921 u16 next_seid = 0; 14922 int ret = 0; 14923 u8 *aq_buf; 14924 int i; 14925 14926 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14927 if (!aq_buf) 14928 return -ENOMEM; 14929 14930 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14931 do { 14932 u16 num_reported, num_total; 14933 14934 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14935 I40E_AQ_LARGE_BUF, 14936 &next_seid, NULL); 14937 if (ret) { 14938 dev_info(&pf->pdev->dev, 14939 "get switch config failed err %d aq_err %s\n", 14940 ret, libie_aq_str(pf->hw.aq.asq_last_status)); 14941 kfree(aq_buf); 14942 return -ENOENT; 14943 } 14944 14945 num_reported = le16_to_cpu(sw_config->header.num_reported); 14946 num_total = le16_to_cpu(sw_config->header.num_total); 14947 14948 if (printconfig) 14949 dev_info(&pf->pdev->dev, 14950 "header: %d reported %d total\n", 14951 num_reported, num_total); 14952 14953 for (i = 0; i < num_reported; i++) { 14954 struct i40e_aqc_switch_config_element_resp *ele = 14955 &sw_config->element[i]; 14956 14957 i40e_setup_pf_switch_element(pf, ele, num_reported, 14958 printconfig); 14959 } 14960 } while (next_seid != 0); 14961 14962 kfree(aq_buf); 14963 return ret; 14964} 14965 14966/** 14967 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14968 * @pf: board private structure 14969 * @reinit: if the Main VSI needs to re-initialized. 14970 * @lock_acquired: indicates whether or not the lock has been acquired 14971 * 14972 * Returns 0 on success, negative value on failure 14973 **/ 14974static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired) 14975{ 14976 struct i40e_vsi *main_vsi; 14977 u16 flags = 0; 14978 int ret; 14979 14980 /* find out what's out there already */ 14981 ret = i40e_fetch_switch_configuration(pf, false); 14982 if (ret) { 14983 dev_info(&pf->pdev->dev, 14984 "couldn't fetch switch config, err %pe aq_err %s\n", 14985 ERR_PTR(ret), libie_aq_str(pf->hw.aq.asq_last_status)); 14986 return ret; 14987 } 14988 i40e_pf_reset_stats(pf); 14989 14990 /* set the switch config bit for the whole device to 14991 * support limited promisc or true promisc 14992 * when user requests promisc. The default is limited 14993 * promisc. 14994 */ 14995 14996 if ((pf->hw.pf_id == 0) && 14997 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) { 14998 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14999 pf->last_sw_conf_flags = flags; 15000 } 15001 15002 if (pf->hw.pf_id == 0) { 15003 u16 valid_flags; 15004 15005 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15006 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 15007 NULL); 15008 if (ret && pf->hw.aq.asq_last_status != LIBIE_AQ_RC_ESRCH) { 15009 dev_info(&pf->pdev->dev, 15010 "couldn't set switch config bits, err %pe aq_err %s\n", 15011 ERR_PTR(ret), 15012 libie_aq_str(pf->hw.aq.asq_last_status)); 15013 /* not a fatal problem, just keep going */ 15014 } 15015 pf->last_sw_conf_valid_flags = valid_flags; 15016 } 15017 15018 /* first time setup */ 15019 main_vsi = i40e_pf_get_main_vsi(pf); 15020 if (!main_vsi || reinit) { 15021 struct i40e_veb *veb; 15022 u16 uplink_seid; 15023 15024 /* Set up the PF VSI associated with the PF's main VSI 15025 * that is already in the HW switch 15026 */ 15027 veb = i40e_pf_get_main_veb(pf); 15028 if (veb) 15029 uplink_seid = veb->seid; 15030 else 15031 uplink_seid = pf->mac_seid; 15032 if (!main_vsi) 15033 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, 15034 uplink_seid, 0); 15035 else if (reinit) 15036 main_vsi = i40e_vsi_reinit_setup(main_vsi); 15037 if (!main_vsi) { 15038 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 15039 i40e_cloud_filter_exit(pf); 15040 i40e_fdir_teardown(pf); 15041 return -EAGAIN; 15042 } 15043 } else { 15044 /* force a reset of TC and queue layout configurations */ 15045 main_vsi->seid = pf->main_vsi_seid; 15046 i40e_vsi_reconfig_tc(main_vsi); 15047 } 15048 i40e_vlan_stripping_disable(main_vsi); 15049 15050 i40e_fdir_sb_setup(pf); 15051 15052 /* Setup static PF queue filter control settings */ 15053 ret = i40e_setup_pf_filter_control(pf); 15054 if (ret) { 15055 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 15056 ret); 15057 /* Failure here should not stop continuing other steps */ 15058 } 15059 15060 /* enable RSS in the HW, even for only one queue, as the stack can use 15061 * the hash 15062 */ 15063 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15064 i40e_pf_config_rss(pf); 15065 15066 /* fill in link information and enable LSE reporting */ 15067 i40e_link_event(pf); 15068 15069 i40e_ptp_init(pf); 15070 15071 if (!lock_acquired) 15072 rtnl_lock(); 15073 15074 /* repopulate tunnel port filters */ 15075 udp_tunnel_nic_reset_ntf(main_vsi->netdev); 15076 15077 if (!lock_acquired) 15078 rtnl_unlock(); 15079 15080 return ret; 15081} 15082 15083/** 15084 * i40e_determine_queue_usage - Work out queue distribution 15085 * @pf: board private structure 15086 **/ 15087static void i40e_determine_queue_usage(struct i40e_pf *pf) 15088{ 15089 int queues_left; 15090 int q_max; 15091 15092 pf->num_lan_qps = 0; 15093 15094 /* Find the max queues to be put into basic use. We'll always be 15095 * using TC0, whether or not DCB is running, and TC0 will get the 15096 * big RSS set. 15097 */ 15098 queues_left = pf->hw.func_caps.num_tx_qp; 15099 15100 if ((queues_left == 1) || 15101 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 15102 /* one qp for PF, no queues for anything else */ 15103 queues_left = 0; 15104 pf->alloc_rss_size = pf->num_lan_qps = 1; 15105 15106 /* make sure all the fancies are disabled */ 15107 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15108 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15109 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15110 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15111 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15112 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15113 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 15114 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15115 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15116 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) && 15117 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && 15118 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) && 15119 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) { 15120 /* one qp for PF */ 15121 pf->alloc_rss_size = pf->num_lan_qps = 1; 15122 queues_left -= pf->num_lan_qps; 15123 15124 clear_bit(I40E_FLAG_RSS_ENA, pf->flags); 15125 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15126 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15127 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags); 15128 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15129 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags); 15130 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15131 } else { 15132 /* Not enough queues for all TCs */ 15133 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) && 15134 queues_left < I40E_MAX_TRAFFIC_CLASS) { 15135 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15136 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15137 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 15138 } 15139 15140 /* limit lan qps to the smaller of qps, cpus or msix */ 15141 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 15142 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 15143 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 15144 pf->num_lan_qps = q_max; 15145 15146 queues_left -= pf->num_lan_qps; 15147 } 15148 15149 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15150 if (queues_left > 1) { 15151 queues_left -= 1; /* save 1 queue for FD */ 15152 } else { 15153 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags); 15154 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags); 15155 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 15156 } 15157 } 15158 15159 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15160 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 15161 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 15162 (queues_left / pf->num_vf_qps)); 15163 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 15164 } 15165 15166 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && 15167 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 15168 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 15169 (queues_left / pf->num_vmdq_qps)); 15170 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 15171 } 15172 15173 pf->queues_left = queues_left; 15174 dev_dbg(&pf->pdev->dev, 15175 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 15176 pf->hw.func_caps.num_tx_qp, 15177 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags), 15178 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 15179 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 15180 queues_left); 15181} 15182 15183/** 15184 * i40e_setup_pf_filter_control - Setup PF static filter control 15185 * @pf: PF to be setup 15186 * 15187 * i40e_setup_pf_filter_control sets up a PF's initial filter control 15188 * settings. If PE/FCoE are enabled then it will also set the per PF 15189 * based filter sizes required for them. It also enables Flow director, 15190 * ethertype and macvlan type filter settings for the pf. 15191 * 15192 * Returns 0 on success, negative on failure 15193 **/ 15194static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 15195{ 15196 struct i40e_filter_control_settings *settings = &pf->filter_settings; 15197 15198 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 15199 15200 /* Flow Director is enabled */ 15201 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) || 15202 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15203 settings->enable_fdir = true; 15204 15205 /* Ethtype and MACVLAN filters enabled for PF */ 15206 settings->enable_ethtype = true; 15207 settings->enable_macvlan = true; 15208 15209 if (i40e_set_filter_control(&pf->hw, settings)) 15210 return -ENOENT; 15211 15212 return 0; 15213} 15214 15215#define INFO_STRING_LEN 255 15216#define REMAIN(__x) (INFO_STRING_LEN - (__x)) 15217static void i40e_print_features(struct i40e_pf *pf) 15218{ 15219 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 15220 struct i40e_hw *hw = &pf->hw; 15221 char *buf; 15222 int i; 15223 15224 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 15225 if (!buf) 15226 return; 15227 15228 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 15229#ifdef CONFIG_PCI_IOV 15230 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 15231#endif 15232 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 15233 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs); 15234 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags)) 15235 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 15236 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags)) 15237 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 15238 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) { 15239 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 15240 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 15241 } 15242 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) 15243 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 15244 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 15245 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 15246 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags)) 15247 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 15248 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) 15249 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 15250 else 15251 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 15252 15253 dev_info(&pf->pdev->dev, "%s\n", buf); 15254 kfree(buf); 15255 WARN_ON(i > INFO_STRING_LEN); 15256} 15257 15258/** 15259 * i40e_get_platform_mac_addr - get platform-specific MAC address 15260 * @pdev: PCI device information struct 15261 * @pf: board private structure 15262 * 15263 * Look up the MAC address for the device. First we'll try 15264 * eth_platform_get_mac_address, which will check Open Firmware, or arch 15265 * specific fallback. Otherwise, we'll default to the stored value in 15266 * firmware. 15267 **/ 15268static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 15269{ 15270 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 15271 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 15272} 15273 15274/** 15275 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 15276 * @fec_cfg: FEC option to set in flags 15277 * @flags: ptr to flags in which we set FEC option 15278 **/ 15279void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags) 15280{ 15281 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) { 15282 set_bit(I40E_FLAG_RS_FEC, flags); 15283 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15284 } 15285 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 15286 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 15287 set_bit(I40E_FLAG_RS_FEC, flags); 15288 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15289 } 15290 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 15291 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 15292 set_bit(I40E_FLAG_BASE_R_FEC, flags); 15293 clear_bit(I40E_FLAG_RS_FEC, flags); 15294 } 15295 if (fec_cfg == 0) { 15296 clear_bit(I40E_FLAG_RS_FEC, flags); 15297 clear_bit(I40E_FLAG_BASE_R_FEC, flags); 15298 } 15299} 15300 15301/** 15302 * i40e_check_recovery_mode - check if we are running transition firmware 15303 * @pf: board private structure 15304 * 15305 * Check registers indicating the firmware runs in recovery mode. Sets the 15306 * appropriate driver state. 15307 * 15308 * Returns true if the recovery mode was detected, false otherwise 15309 **/ 15310static bool i40e_check_recovery_mode(struct i40e_pf *pf) 15311{ 15312 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 15313 15314 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 15315 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 15316 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 15317 set_bit(__I40E_RECOVERY_MODE, pf->state); 15318 15319 return true; 15320 } 15321 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15322 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 15323 15324 return false; 15325} 15326 15327/** 15328 * i40e_pf_loop_reset - perform reset in a loop. 15329 * @pf: board private structure 15330 * 15331 * This function is useful when a NIC is about to enter recovery mode. 15332 * When a NIC's internal data structures are corrupted the NIC's 15333 * firmware is going to enter recovery mode. 15334 * Right after a POR it takes about 7 minutes for firmware to enter 15335 * recovery mode. Until that time a NIC is in some kind of intermediate 15336 * state. After that time period the NIC almost surely enters 15337 * recovery mode. The only way for a driver to detect intermediate 15338 * state is to issue a series of pf-resets and check a return value. 15339 * If a PF reset returns success then the firmware could be in recovery 15340 * mode so the caller of this code needs to check for recovery mode 15341 * if this function returns success. There is a little chance that 15342 * firmware will hang in intermediate state forever. 15343 * Since waiting 7 minutes is quite a lot of time this function waits 15344 * 10 seconds and then gives up by returning an error. 15345 * 15346 * Return 0 on success, negative on failure. 15347 **/ 15348static int i40e_pf_loop_reset(struct i40e_pf *pf) 15349{ 15350 /* wait max 10 seconds for PF reset to succeed */ 15351 const unsigned long time_end = jiffies + 10 * HZ; 15352 struct i40e_hw *hw = &pf->hw; 15353 int ret; 15354 15355 ret = i40e_pf_reset(hw); 15356 while (ret != 0 && time_before(jiffies, time_end)) { 15357 usleep_range(10000, 20000); 15358 ret = i40e_pf_reset(hw); 15359 } 15360 15361 if (ret == 0) 15362 pf->pfr_count++; 15363 else 15364 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 15365 15366 return ret; 15367} 15368 15369/** 15370 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 15371 * @pf: board private structure 15372 * 15373 * Check FW registers to determine if FW issued unexpected EMP Reset. 15374 * Every time when unexpected EMP Reset occurs the FW increments 15375 * a counter of unexpected EMP Resets. When the counter reaches 10 15376 * the FW should enter the Recovery mode 15377 * 15378 * Returns true if FW issued unexpected EMP Reset 15379 **/ 15380static bool i40e_check_fw_empr(struct i40e_pf *pf) 15381{ 15382 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 15383 I40E_GL_FWSTS_FWS1B_MASK; 15384 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 15385 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 15386} 15387 15388/** 15389 * i40e_handle_resets - handle EMP resets and PF resets 15390 * @pf: board private structure 15391 * 15392 * Handle both EMP resets and PF resets and conclude whether there are 15393 * any issues regarding these resets. If there are any issues then 15394 * generate log entry. 15395 * 15396 * Return 0 if NIC is healthy or negative value when there are issues 15397 * with resets 15398 **/ 15399static int i40e_handle_resets(struct i40e_pf *pf) 15400{ 15401 const int pfr = i40e_pf_loop_reset(pf); 15402 const bool is_empr = i40e_check_fw_empr(pf); 15403 15404 if (is_empr || pfr != 0) 15405 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 15406 15407 return is_empr ? -EIO : pfr; 15408} 15409 15410/** 15411 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 15412 * @pf: board private structure 15413 * @hw: ptr to the hardware info 15414 * 15415 * This function does a minimal setup of all subsystems needed for running 15416 * recovery mode. 15417 * 15418 * Returns 0 on success, negative on failure 15419 **/ 15420static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 15421{ 15422 struct i40e_vsi *vsi; 15423 int err; 15424 int v_idx; 15425 15426 pci_set_drvdata(pf->pdev, pf); 15427 pci_save_state(pf->pdev); 15428 15429 /* set up periodic task facility */ 15430 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15431 pf->service_timer_period = HZ; 15432 15433 INIT_WORK(&pf->service_task, i40e_service_task); 15434 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15435 15436 err = i40e_init_interrupt_scheme(pf); 15437 if (err) 15438 goto err_switch_setup; 15439 15440 /* The number of VSIs reported by the FW is the minimum guaranteed 15441 * to us; HW supports far more and we share the remaining pool with 15442 * the other PFs. We allocate space for more than the guarantee with 15443 * the understanding that we might not get them all later. 15444 */ 15445 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15446 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15447 else 15448 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15449 15450 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 15451 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15452 GFP_KERNEL); 15453 if (!pf->vsi) { 15454 err = -ENOMEM; 15455 goto err_switch_setup; 15456 } 15457 15458 /* We allocate one VSI which is needed as absolute minimum 15459 * in order to register the netdev 15460 */ 15461 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 15462 if (v_idx < 0) { 15463 err = v_idx; 15464 goto err_switch_setup; 15465 } 15466 pf->lan_vsi = v_idx; 15467 vsi = pf->vsi[v_idx]; 15468 if (!vsi) { 15469 err = -EFAULT; 15470 goto err_switch_setup; 15471 } 15472 vsi->alloc_queue_pairs = 1; 15473 err = i40e_config_netdev(vsi); 15474 if (err) 15475 goto err_switch_setup; 15476 err = register_netdev(vsi->netdev); 15477 if (err) 15478 goto err_switch_setup; 15479 vsi->netdev_registered = true; 15480 i40e_dbg_pf_init(pf); 15481 15482 err = i40e_setup_misc_vector_for_recovery_mode(pf); 15483 if (err) 15484 goto err_switch_setup; 15485 15486 /* tell the firmware that we're starting */ 15487 i40e_send_version(pf); 15488 15489 /* since everything's happy, start the service_task timer */ 15490 mod_timer(&pf->service_timer, 15491 round_jiffies(jiffies + pf->service_timer_period)); 15492 15493 return 0; 15494 15495err_switch_setup: 15496 i40e_reset_interrupt_capability(pf); 15497 timer_shutdown_sync(&pf->service_timer); 15498 i40e_shutdown_adminq(hw); 15499 iounmap(hw->hw_addr); 15500 pci_release_mem_regions(pf->pdev); 15501 pci_disable_device(pf->pdev); 15502 i40e_free_pf(pf); 15503 15504 return err; 15505} 15506 15507/** 15508 * i40e_set_subsystem_device_id - set subsystem device id 15509 * @hw: pointer to the hardware info 15510 * 15511 * Set PCI subsystem device id either from a pci_dev structure or 15512 * a specific FW register. 15513 **/ 15514static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw) 15515{ 15516 struct i40e_pf *pf = i40e_hw_to_pf(hw); 15517 15518 hw->subsystem_device_id = pf->pdev->subsystem_device ? 15519 pf->pdev->subsystem_device : 15520 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX); 15521} 15522 15523/** 15524 * i40e_probe - Device initialization routine 15525 * @pdev: PCI device information struct 15526 * @ent: entry in i40e_pci_tbl 15527 * 15528 * i40e_probe initializes a PF identified by a pci_dev structure. 15529 * The OS initialization, configuring of the PF private structure, 15530 * and a hardware reset occur. 15531 * 15532 * Returns 0 on success, negative on failure 15533 **/ 15534static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 15535{ 15536 struct i40e_aq_get_phy_abilities_resp abilities; 15537#ifdef CONFIG_I40E_DCB 15538 enum i40e_get_fw_lldp_status_resp lldp_status; 15539#endif /* CONFIG_I40E_DCB */ 15540 struct i40e_vsi *vsi; 15541 struct i40e_pf *pf; 15542 struct i40e_hw *hw; 15543 u16 wol_nvm_bits; 15544 char nvm_ver[32]; 15545 u16 link_status; 15546#ifdef CONFIG_I40E_DCB 15547 int status; 15548#endif /* CONFIG_I40E_DCB */ 15549 int err; 15550 u32 val; 15551 15552 err = pci_enable_device_mem(pdev); 15553 if (err) 15554 return err; 15555 15556 /* set up for high or low dma */ 15557 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 15558 if (err) { 15559 dev_err(&pdev->dev, 15560 "DMA configuration failed: 0x%x\n", err); 15561 goto err_dma; 15562 } 15563 15564 /* set up pci connections */ 15565 err = pci_request_mem_regions(pdev, i40e_driver_name); 15566 if (err) { 15567 dev_info(&pdev->dev, 15568 "pci_request_selected_regions failed %d\n", err); 15569 goto err_pci_reg; 15570 } 15571 15572 pci_set_master(pdev); 15573 15574 /* Now that we have a PCI connection, we need to do the 15575 * low level device setup. This is primarily setting up 15576 * the Admin Queue structures and then querying for the 15577 * device's current profile information. 15578 */ 15579 pf = i40e_alloc_pf(&pdev->dev); 15580 if (!pf) { 15581 err = -ENOMEM; 15582 goto err_pf_alloc; 15583 } 15584 pf->next_vsi = 0; 15585 pf->pdev = pdev; 15586 set_bit(__I40E_DOWN, pf->state); 15587 15588 hw = &pf->hw; 15589 15590 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 15591 I40E_MAX_CSR_SPACE); 15592 /* We believe that the highest register to read is 15593 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 15594 * is not less than that before mapping to prevent a 15595 * kernel panic. 15596 */ 15597 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 15598 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 15599 pf->ioremap_len); 15600 err = -ENOMEM; 15601 goto err_ioremap; 15602 } 15603 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 15604 if (!hw->hw_addr) { 15605 err = -EIO; 15606 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 15607 (unsigned int)pci_resource_start(pdev, 0), 15608 pf->ioremap_len, err); 15609 goto err_ioremap; 15610 } 15611 hw->vendor_id = pdev->vendor; 15612 hw->device_id = pdev->device; 15613 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 15614 hw->subsystem_vendor_id = pdev->subsystem_vendor; 15615 i40e_set_subsystem_device_id(hw); 15616 hw->bus.device = PCI_SLOT(pdev->devfn); 15617 hw->bus.func = PCI_FUNC(pdev->devfn); 15618 hw->bus.bus_id = pdev->bus->number; 15619 15620 /* Select something other than the 802.1ad ethertype for the 15621 * switch to use internally and drop on ingress. 15622 */ 15623 hw->switch_tag = 0xffff; 15624 hw->first_tag = ETH_P_8021AD; 15625 hw->second_tag = ETH_P_8021Q; 15626 15627 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 15628 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 15629 INIT_LIST_HEAD(&pf->ddp_old_prof); 15630 15631 /* set up the locks for the AQ, do this only once in probe 15632 * and destroy them only once in remove 15633 */ 15634 mutex_init(&hw->aq.asq_mutex); 15635 mutex_init(&hw->aq.arq_mutex); 15636 15637 pf->msg_enable = netif_msg_init(debug, 15638 NETIF_MSG_DRV | 15639 NETIF_MSG_PROBE | 15640 NETIF_MSG_LINK); 15641 if (debug < -1) 15642 pf->hw.debug_mask = debug; 15643 15644 /* do a special CORER for clearing PXE mode once at init */ 15645 if (hw->revision_id == 0 && 15646 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 15647 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 15648 i40e_flush(hw); 15649 msleep(200); 15650 pf->corer_count++; 15651 15652 i40e_clear_pxe_mode(hw); 15653 } 15654 15655 /* Reset here to make sure all is clean and to define PF 'n' */ 15656 i40e_clear_hw(hw); 15657 15658 err = i40e_set_mac_type(hw); 15659 if (err) { 15660 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15661 err); 15662 goto err_pf_reset; 15663 } 15664 15665 err = i40e_handle_resets(pf); 15666 if (err) 15667 goto err_pf_reset; 15668 15669 i40e_check_recovery_mode(pf); 15670 15671 if (is_kdump_kernel()) { 15672 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN; 15673 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN; 15674 } else { 15675 hw->aq.num_arq_entries = I40E_AQ_LEN; 15676 hw->aq.num_asq_entries = I40E_AQ_LEN; 15677 } 15678 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15679 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15680 15681 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 15682 "%s-%s:misc", 15683 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 15684 15685 err = i40e_init_shared_code(hw); 15686 if (err) { 15687 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15688 err); 15689 goto err_pf_reset; 15690 } 15691 15692 /* set up a default setting for link flow control */ 15693 pf->hw.fc.requested_mode = I40E_FC_NONE; 15694 15695 err = i40e_init_adminq(hw); 15696 if (err) { 15697 if (err == -EIO) 15698 dev_info(&pdev->dev, 15699 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 15700 hw->aq.api_maj_ver, 15701 hw->aq.api_min_ver, 15702 I40E_FW_API_VERSION_MAJOR, 15703 I40E_FW_MINOR_VERSION(hw)); 15704 else 15705 dev_info(&pdev->dev, 15706 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 15707 15708 goto err_pf_reset; 15709 } 15710 i40e_get_oem_version(hw); 15711 i40e_get_pba_string(hw); 15712 15713 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 15714 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver)); 15715 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 15716 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 15717 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver, 15718 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id, 15719 hw->subsystem_device_id); 15720 15721 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR, 15722 I40E_FW_MINOR_VERSION(hw) + 1)) 15723 dev_dbg(&pdev->dev, 15724 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n", 15725 hw->aq.api_maj_ver, 15726 hw->aq.api_min_ver, 15727 I40E_FW_API_VERSION_MAJOR, 15728 I40E_FW_MINOR_VERSION(hw)); 15729 else if (i40e_is_aq_api_ver_lt(hw, 1, 4)) 15730 dev_info(&pdev->dev, 15731 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 15732 hw->aq.api_maj_ver, 15733 hw->aq.api_min_ver, 15734 I40E_FW_API_VERSION_MAJOR, 15735 I40E_FW_MINOR_VERSION(hw)); 15736 15737 i40e_verify_eeprom(pf); 15738 15739 /* Rev 0 hardware was never productized */ 15740 if (hw->revision_id < 1) 15741 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 15742 15743 i40e_clear_pxe_mode(hw); 15744 15745 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 15746 if (err) 15747 goto err_adminq_setup; 15748 15749 err = i40e_sw_init(pf); 15750 if (err) { 15751 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 15752 goto err_sw_init; 15753 } 15754 15755 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15756 return i40e_init_recovery_mode(pf, hw); 15757 15758 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 15759 hw->func_caps.num_rx_qp, 0, 0); 15760 if (err) { 15761 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 15762 goto err_init_lan_hmc; 15763 } 15764 15765 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 15766 if (err) { 15767 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 15768 err = -ENOENT; 15769 goto err_configure_lan_hmc; 15770 } 15771 15772 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 15773 * Ignore error return codes because if it was already disabled via 15774 * hardware settings this will fail 15775 */ 15776 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) { 15777 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15778 i40e_aq_stop_lldp(hw, true, false, NULL); 15779 } 15780 15781 /* allow a platform config to override the HW addr */ 15782 i40e_get_platform_mac_addr(pdev, pf); 15783 15784 if (!is_valid_ether_addr(hw->mac.addr)) { 15785 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15786 err = -EIO; 15787 goto err_mac_addr; 15788 } 15789 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15790 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15791 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15792 if (is_valid_ether_addr(hw->mac.port_addr)) 15793 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps); 15794 15795 i40e_ptp_alloc_pins(pf); 15796 pci_set_drvdata(pdev, pf); 15797 pci_save_state(pdev); 15798 15799#ifdef CONFIG_I40E_DCB 15800 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status); 15801 (!status && 15802 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ? 15803 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) : 15804 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)); 15805 dev_info(&pdev->dev, 15806 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ? 15807 "FW LLDP is disabled\n" : 15808 "FW LLDP is enabled\n"); 15809 15810 /* Enable FW to write default DCB config on link-up */ 15811 i40e_aq_set_dcb_parameters(hw, true, NULL); 15812 15813 err = i40e_init_pf_dcb(pf); 15814 if (err) { 15815 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15816 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags); 15817 clear_bit(I40E_FLAG_DCB_ENA, pf->flags); 15818 /* Continue without DCB enabled */ 15819 } 15820#endif /* CONFIG_I40E_DCB */ 15821 15822 /* set up periodic task facility */ 15823 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15824 pf->service_timer_period = HZ; 15825 15826 INIT_WORK(&pf->service_task, i40e_service_task); 15827 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15828 15829 /* NVM bit on means WoL disabled for the port */ 15830 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15831 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15832 pf->wol_en = false; 15833 else 15834 pf->wol_en = true; 15835 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15836 15837 /* set up the main switch operations */ 15838 i40e_determine_queue_usage(pf); 15839 err = i40e_init_interrupt_scheme(pf); 15840 if (err) 15841 goto err_switch_setup; 15842 15843 /* Reduce Tx and Rx pairs for kdump 15844 * When MSI-X is enabled, it's not allowed to use more TC queue 15845 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus 15846 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1. 15847 */ 15848 if (is_kdump_kernel()) 15849 pf->num_lan_msix = 1; 15850 15851 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; 15852 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; 15853 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; 15854 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; 15855 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 15856 UDP_TUNNEL_TYPE_GENEVE; 15857 15858 /* The number of VSIs reported by the FW is the minimum guaranteed 15859 * to us; HW supports far more and we share the remaining pool with 15860 * the other PFs. We allocate space for more than the guarantee with 15861 * the understanding that we might not get them all later. 15862 */ 15863 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15864 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15865 else 15866 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15867 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 15868 dev_warn(&pf->pdev->dev, 15869 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 15870 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 15871 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 15872 } 15873 15874 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15875 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15876 GFP_KERNEL); 15877 if (!pf->vsi) { 15878 err = -ENOMEM; 15879 goto err_switch_setup; 15880 } 15881 15882#ifdef CONFIG_PCI_IOV 15883 /* prep for VF support */ 15884 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15885 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 15886 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15887 if (pci_num_vf(pdev)) 15888 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags); 15889 } 15890#endif 15891 err = i40e_setup_pf_switch(pf, false, false); 15892 if (err) { 15893 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15894 goto err_vsis; 15895 } 15896 15897 vsi = i40e_pf_get_main_vsi(pf); 15898 INIT_LIST_HEAD(&vsi->ch_list); 15899 15900 /* if FDIR VSI was set up, start it now */ 15901 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 15902 if (vsi) 15903 i40e_vsi_open(vsi); 15904 15905 /* The driver only wants link up/down and module qualification 15906 * reports from firmware. Note the negative logic. 15907 */ 15908 err = i40e_aq_set_phy_int_mask(&pf->hw, 15909 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15910 I40E_AQ_EVENT_MEDIA_NA | 15911 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15912 if (err) 15913 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 15914 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 15915 15916 /* VF MDD event logs are rate limited to one second intervals */ 15917 ratelimit_state_init(&pf->mdd_message_rate_limit, 1 * HZ, 1); 15918 15919 /* Reconfigure hardware for allowing smaller MSS in the case 15920 * of TSO, so that we avoid the MDD being fired and causing 15921 * a reset in the case of small MSS+TSO. 15922 */ 15923 val = rd32(hw, I40E_REG_MSS); 15924 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15925 val &= ~I40E_REG_MSS_MIN_MASK; 15926 val |= I40E_64BYTE_MSS; 15927 wr32(hw, I40E_REG_MSS, val); 15928 } 15929 15930 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) { 15931 msleep(75); 15932 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15933 if (err) 15934 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 15935 ERR_PTR(err), 15936 libie_aq_str(pf->hw.aq.asq_last_status)); 15937 } 15938 /* The main driver is (mostly) up and happy. We need to set this state 15939 * before setting up the misc vector or we get a race and the vector 15940 * ends up disabled forever. 15941 */ 15942 clear_bit(__I40E_DOWN, pf->state); 15943 15944 /* In case of MSIX we are going to setup the misc vector right here 15945 * to handle admin queue events etc. In case of legacy and MSI 15946 * the misc functionality and queue processing is combined in 15947 * the same vector and that gets setup at open. 15948 */ 15949 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) { 15950 err = i40e_setup_misc_vector(pf); 15951 if (err) { 15952 dev_info(&pdev->dev, 15953 "setup of misc vector failed: %d\n", err); 15954 i40e_cloud_filter_exit(pf); 15955 i40e_fdir_teardown(pf); 15956 goto err_vsis; 15957 } 15958 } 15959 15960#ifdef CONFIG_PCI_IOV 15961 /* prep for VF support */ 15962 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) && 15963 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && 15964 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15965 /* disable link interrupts for VFs */ 15966 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15967 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15968 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15969 i40e_flush(hw); 15970 15971 if (pci_num_vf(pdev)) { 15972 dev_info(&pdev->dev, 15973 "Active VFs found, allocating resources.\n"); 15974 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15975 if (err) 15976 dev_info(&pdev->dev, 15977 "Error %d allocating resources for existing VFs\n", 15978 err); 15979 } 15980 } 15981#endif /* CONFIG_PCI_IOV */ 15982 15983 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 15984 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15985 pf->num_iwarp_msix, 15986 I40E_IWARP_IRQ_PILE_ID); 15987 if (pf->iwarp_base_vector < 0) { 15988 dev_info(&pdev->dev, 15989 "failed to get tracking for %d vectors for IWARP err=%d\n", 15990 pf->num_iwarp_msix, pf->iwarp_base_vector); 15991 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags); 15992 } 15993 } 15994 15995 i40e_dbg_pf_init(pf); 15996 15997 /* tell the firmware that we're starting */ 15998 i40e_send_version(pf); 15999 16000 /* since everything's happy, start the service_task timer */ 16001 mod_timer(&pf->service_timer, 16002 round_jiffies(jiffies + pf->service_timer_period)); 16003 16004 /* add this PF to client device list and launch a client service task */ 16005 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16006 err = i40e_lan_add_device(pf); 16007 if (err) 16008 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 16009 err); 16010 } 16011 16012#define PCI_SPEED_SIZE 8 16013#define PCI_WIDTH_SIZE 8 16014 /* Devices on the IOSF bus do not have this information 16015 * and will report PCI Gen 1 x 1 by default so don't bother 16016 * checking them. 16017 */ 16018 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) { 16019 char speed[PCI_SPEED_SIZE] = "Unknown"; 16020 char width[PCI_WIDTH_SIZE] = "Unknown"; 16021 16022 /* Get the negotiated link width and speed from PCI config 16023 * space 16024 */ 16025 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 16026 &link_status); 16027 16028 i40e_set_pci_config_data(hw, link_status); 16029 16030 switch (hw->bus.speed) { 16031 case i40e_bus_speed_8000: 16032 strscpy(speed, "8.0", PCI_SPEED_SIZE); break; 16033 case i40e_bus_speed_5000: 16034 strscpy(speed, "5.0", PCI_SPEED_SIZE); break; 16035 case i40e_bus_speed_2500: 16036 strscpy(speed, "2.5", PCI_SPEED_SIZE); break; 16037 default: 16038 break; 16039 } 16040 switch (hw->bus.width) { 16041 case i40e_bus_width_pcie_x8: 16042 strscpy(width, "8", PCI_WIDTH_SIZE); break; 16043 case i40e_bus_width_pcie_x4: 16044 strscpy(width, "4", PCI_WIDTH_SIZE); break; 16045 case i40e_bus_width_pcie_x2: 16046 strscpy(width, "2", PCI_WIDTH_SIZE); break; 16047 case i40e_bus_width_pcie_x1: 16048 strscpy(width, "1", PCI_WIDTH_SIZE); break; 16049 default: 16050 break; 16051 } 16052 16053 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 16054 speed, width); 16055 16056 if (hw->bus.width < i40e_bus_width_pcie_x8 || 16057 hw->bus.speed < i40e_bus_speed_8000) { 16058 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 16059 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 16060 } 16061 } 16062 16063 /* get the requested speeds from the fw */ 16064 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 16065 if (err) 16066 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n", 16067 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16068 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 16069 16070 /* set the FEC config due to the board capabilities */ 16071 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags); 16072 16073 /* get the supported phy types from the fw */ 16074 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 16075 if (err) 16076 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n", 16077 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16078 16079#define MAX_FRAME_SIZE_DEFAULT 0x2600 16080 16081 err = i40e_aq_set_mac_config(hw, MAX_FRAME_SIZE_DEFAULT, NULL); 16082 if (err) 16083 dev_warn(&pdev->dev, "set mac config ret = %pe last_status = %s\n", 16084 ERR_PTR(err), libie_aq_str(pf->hw.aq.asq_last_status)); 16085 16086 /* Make sure the MFS is set to the expected value */ 16087 val = rd32(hw, I40E_PRTGL_SAH); 16088 FIELD_MODIFY(I40E_PRTGL_SAH_MFS_MASK, &val, MAX_FRAME_SIZE_DEFAULT); 16089 wr32(hw, I40E_PRTGL_SAH, val); 16090 16091 /* Add a filter to drop all Flow control frames from any VSI from being 16092 * transmitted. By doing so we stop a malicious VF from sending out 16093 * PAUSE or PFC frames and potentially controlling traffic for other 16094 * PF/VF VSIs. 16095 * The FW can still send Flow control frames if enabled. 16096 */ 16097 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 16098 pf->main_vsi_seid); 16099 16100 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 16101 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 16102 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps); 16103 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 16104 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps); 16105 /* print a string summarizing features */ 16106 i40e_print_features(pf); 16107 16108 i40e_devlink_register(pf); 16109 16110 return 0; 16111 16112 /* Unwind what we've done if something failed in the setup */ 16113err_vsis: 16114 set_bit(__I40E_DOWN, pf->state); 16115 i40e_clear_interrupt_scheme(pf); 16116 kfree(pf->vsi); 16117err_switch_setup: 16118 i40e_reset_interrupt_capability(pf); 16119 timer_shutdown_sync(&pf->service_timer); 16120err_mac_addr: 16121err_configure_lan_hmc: 16122 (void)i40e_shutdown_lan_hmc(hw); 16123err_init_lan_hmc: 16124 kfree(pf->qp_pile); 16125err_sw_init: 16126err_adminq_setup: 16127err_pf_reset: 16128 iounmap(hw->hw_addr); 16129err_ioremap: 16130 i40e_free_pf(pf); 16131err_pf_alloc: 16132 pci_release_mem_regions(pdev); 16133err_pci_reg: 16134err_dma: 16135 pci_disable_device(pdev); 16136 return err; 16137} 16138 16139/** 16140 * i40e_remove - Device removal routine 16141 * @pdev: PCI device information struct 16142 * 16143 * i40e_remove is called by the PCI subsystem to alert the driver 16144 * that is should release a PCI device. This could be caused by a 16145 * Hot-Plug event, or because the driver is going to be removed from 16146 * memory. 16147 **/ 16148static void i40e_remove(struct pci_dev *pdev) 16149{ 16150 struct i40e_pf *pf = pci_get_drvdata(pdev); 16151 struct i40e_hw *hw = &pf->hw; 16152 struct i40e_vsi *vsi; 16153 struct i40e_veb *veb; 16154 int ret_code; 16155 int i; 16156 16157 i40e_devlink_unregister(pf); 16158 16159 i40e_dbg_pf_exit(pf); 16160 16161 i40e_ptp_stop(pf); 16162 16163 /* Disable RSS in hw */ 16164 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 16165 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 16166 16167 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE 16168 * flags, once they are set, i40e_rebuild should not be called as 16169 * i40e_prep_for_reset always returns early. 16170 */ 16171 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 16172 usleep_range(1000, 2000); 16173 set_bit(__I40E_IN_REMOVE, pf->state); 16174 16175 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) { 16176 set_bit(__I40E_VF_RESETS_DISABLED, pf->state); 16177 i40e_free_vfs(pf); 16178 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags); 16179 } 16180 /* no more scheduling of any task */ 16181 set_bit(__I40E_SUSPENDED, pf->state); 16182 set_bit(__I40E_DOWN, pf->state); 16183 if (pf->service_timer.function) 16184 timer_shutdown_sync(&pf->service_timer); 16185 if (pf->service_task.func) 16186 cancel_work_sync(&pf->service_task); 16187 16188 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 16189 struct i40e_vsi *vsi = pf->vsi[0]; 16190 16191 /* We know that we have allocated only one vsi for this PF, 16192 * it was just for registering netdevice, so the interface 16193 * could be visible in the 'ifconfig' output 16194 */ 16195 unregister_netdev(vsi->netdev); 16196 free_netdev(vsi->netdev); 16197 16198 goto unmap; 16199 } 16200 16201 /* Client close must be called explicitly here because the timer 16202 * has been stopped. 16203 */ 16204 i40e_notify_client_of_netdev_close(pf, false); 16205 16206 i40e_fdir_teardown(pf); 16207 16208 /* If there is a switch structure or any orphans, remove them. 16209 * This will leave only the PF's VSI remaining. 16210 */ 16211 i40e_pf_for_each_veb(pf, i, veb) 16212 if (veb->uplink_seid == pf->mac_seid || 16213 veb->uplink_seid == 0) 16214 i40e_switch_branch_release(veb); 16215 16216 /* Now we can shutdown the PF's VSIs, just before we kill 16217 * adminq and hmc. 16218 */ 16219 i40e_pf_for_each_vsi(pf, i, vsi) { 16220 i40e_vsi_close(vsi); 16221 i40e_vsi_release(vsi); 16222 pf->vsi[i] = NULL; 16223 } 16224 16225 i40e_cloud_filter_exit(pf); 16226 16227 /* remove attached clients */ 16228 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) { 16229 ret_code = i40e_lan_del_device(pf); 16230 if (ret_code) 16231 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 16232 ret_code); 16233 } 16234 16235 /* shutdown and destroy the HMC */ 16236 if (hw->hmc.hmc_obj) { 16237 ret_code = i40e_shutdown_lan_hmc(hw); 16238 if (ret_code) 16239 dev_warn(&pdev->dev, 16240 "Failed to destroy the HMC resources: %d\n", 16241 ret_code); 16242 } 16243 16244unmap: 16245 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16246 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16247 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16248 free_irq(pf->pdev->irq, pf); 16249 16250 /* shutdown the adminq */ 16251 i40e_shutdown_adminq(hw); 16252 16253 /* destroy the locks only once, here */ 16254 mutex_destroy(&hw->aq.arq_mutex); 16255 mutex_destroy(&hw->aq.asq_mutex); 16256 16257 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 16258 rtnl_lock(); 16259 i40e_clear_interrupt_scheme(pf); 16260 i40e_pf_for_each_vsi(pf, i, vsi) { 16261 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 16262 i40e_vsi_clear_rings(vsi); 16263 16264 i40e_vsi_clear(vsi); 16265 pf->vsi[i] = NULL; 16266 } 16267 rtnl_unlock(); 16268 16269 i40e_pf_for_each_veb(pf, i, veb) { 16270 kfree(veb); 16271 pf->veb[i] = NULL; 16272 } 16273 16274 kfree(pf->qp_pile); 16275 kfree(pf->vsi); 16276 16277 iounmap(hw->hw_addr); 16278 i40e_free_pf(pf); 16279 pci_release_mem_regions(pdev); 16280 16281 pci_disable_device(pdev); 16282} 16283 16284/** 16285 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 16286 * using the mac_address_write admin q function 16287 * @pf: pointer to i40e_pf struct 16288 **/ 16289static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 16290{ 16291 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf); 16292 struct i40e_hw *hw = &pf->hw; 16293 u8 mac_addr[6]; 16294 u16 flags = 0; 16295 int ret; 16296 16297 /* Get current MAC address in case it's an LAA */ 16298 if (main_vsi && main_vsi->netdev) { 16299 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr); 16300 } else { 16301 dev_err(&pf->pdev->dev, 16302 "Failed to retrieve MAC address; using default\n"); 16303 ether_addr_copy(mac_addr, hw->mac.addr); 16304 } 16305 16306 /* The FW expects the mac address write cmd to first be called with 16307 * one of these flags before calling it again with the multicast 16308 * enable flags. 16309 */ 16310 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 16311 16312 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 16313 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 16314 16315 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16316 if (ret) { 16317 dev_err(&pf->pdev->dev, 16318 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 16319 return; 16320 } 16321 16322 flags = I40E_AQC_MC_MAG_EN 16323 | I40E_AQC_WOL_PRESERVE_ON_PFR 16324 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 16325 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16326 if (ret) 16327 dev_err(&pf->pdev->dev, 16328 "Failed to enable Multicast Magic Packet wake up\n"); 16329} 16330 16331/** 16332 * i40e_io_suspend - suspend all IO operations 16333 * @pf: pointer to i40e_pf struct 16334 * 16335 **/ 16336static int i40e_io_suspend(struct i40e_pf *pf) 16337{ 16338 struct i40e_hw *hw = &pf->hw; 16339 16340 set_bit(__I40E_DOWN, pf->state); 16341 16342 /* Ensure service task will not be running */ 16343 timer_delete_sync(&pf->service_timer); 16344 cancel_work_sync(&pf->service_task); 16345 16346 /* Client close must be called explicitly here because the timer 16347 * has been stopped. 16348 */ 16349 i40e_notify_client_of_netdev_close(pf, false); 16350 16351 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16352 pf->wol_en) 16353 i40e_enable_mc_magic_wake(pf); 16354 16355 /* Since we're going to destroy queues during the 16356 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16357 * whole section 16358 */ 16359 rtnl_lock(); 16360 16361 i40e_prep_for_reset(pf); 16362 16363 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16364 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16365 16366 /* Clear the interrupt scheme and release our IRQs so that the system 16367 * can safely hibernate even when there are a large number of CPUs. 16368 * Otherwise hibernation might fail when mapping all the vectors back 16369 * to CPU0. 16370 */ 16371 i40e_clear_interrupt_scheme(pf); 16372 16373 rtnl_unlock(); 16374 16375 return 0; 16376} 16377 16378/** 16379 * i40e_io_resume - resume IO operations 16380 * @pf: pointer to i40e_pf struct 16381 * 16382 **/ 16383static int i40e_io_resume(struct i40e_pf *pf) 16384{ 16385 struct device *dev = &pf->pdev->dev; 16386 int err; 16387 16388 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 16389 * since we're going to be restoring queues 16390 */ 16391 rtnl_lock(); 16392 16393 /* We cleared the interrupt scheme when we suspended, so we need to 16394 * restore it now to resume device functionality. 16395 */ 16396 err = i40e_restore_interrupt_scheme(pf); 16397 if (err) { 16398 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 16399 err); 16400 } 16401 16402 clear_bit(__I40E_DOWN, pf->state); 16403 i40e_reset_and_rebuild(pf, false, true); 16404 16405 rtnl_unlock(); 16406 16407 /* Clear suspended state last after everything is recovered */ 16408 clear_bit(__I40E_SUSPENDED, pf->state); 16409 16410 /* Restart the service task */ 16411 mod_timer(&pf->service_timer, 16412 round_jiffies(jiffies + pf->service_timer_period)); 16413 16414 return 0; 16415} 16416 16417/** 16418 * i40e_pci_error_detected - warning that something funky happened in PCI land 16419 * @pdev: PCI device information struct 16420 * @error: the type of PCI error 16421 * 16422 * Called to warn that something happened and the error handling steps 16423 * are in progress. Allows the driver to quiesce things, be ready for 16424 * remediation. 16425 **/ 16426static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 16427 pci_channel_state_t error) 16428{ 16429 struct i40e_pf *pf = pci_get_drvdata(pdev); 16430 16431 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 16432 16433 if (!pf) { 16434 dev_info(&pdev->dev, 16435 "Cannot recover - error happened during device probe\n"); 16436 return PCI_ERS_RESULT_DISCONNECT; 16437 } 16438 16439 /* shutdown all operations */ 16440 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16441 i40e_io_suspend(pf); 16442 16443 /* Request a slot reset */ 16444 return PCI_ERS_RESULT_NEED_RESET; 16445} 16446 16447/** 16448 * i40e_pci_error_slot_reset - a PCI slot reset just happened 16449 * @pdev: PCI device information struct 16450 * 16451 * Called to find if the driver can work with the device now that 16452 * the pci slot has been reset. If a basic connection seems good 16453 * (registers are readable and have sane content) then return a 16454 * happy little PCI_ERS_RESULT_xxx. 16455 **/ 16456static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 16457{ 16458 struct i40e_pf *pf = pci_get_drvdata(pdev); 16459 pci_ers_result_t result; 16460 u32 reg; 16461 16462 dev_dbg(&pdev->dev, "%s\n", __func__); 16463 /* enable I/O and memory of the device */ 16464 if (pci_enable_device(pdev)) { 16465 dev_info(&pdev->dev, 16466 "Cannot re-enable PCI device after reset.\n"); 16467 result = PCI_ERS_RESULT_DISCONNECT; 16468 } else { 16469 pci_set_master(pdev); 16470 pci_restore_state(pdev); 16471 pci_wake_from_d3(pdev, false); 16472 16473 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 16474 if (reg == 0) 16475 result = PCI_ERS_RESULT_RECOVERED; 16476 else 16477 result = PCI_ERS_RESULT_DISCONNECT; 16478 } 16479 16480 return result; 16481} 16482 16483/** 16484 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 16485 * @pdev: PCI device information struct 16486 */ 16487static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 16488{ 16489 struct i40e_pf *pf = pci_get_drvdata(pdev); 16490 16491 i40e_prep_for_reset(pf); 16492} 16493 16494/** 16495 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 16496 * @pdev: PCI device information struct 16497 */ 16498static void i40e_pci_error_reset_done(struct pci_dev *pdev) 16499{ 16500 struct i40e_pf *pf = pci_get_drvdata(pdev); 16501 16502 if (test_bit(__I40E_IN_REMOVE, pf->state)) 16503 return; 16504 16505 i40e_reset_and_rebuild(pf, false, false); 16506#ifdef CONFIG_PCI_IOV 16507 i40e_restore_all_vfs_msi_state(pdev); 16508#endif /* CONFIG_PCI_IOV */ 16509} 16510 16511/** 16512 * i40e_pci_error_resume - restart operations after PCI error recovery 16513 * @pdev: PCI device information struct 16514 * 16515 * Called to allow the driver to bring things back up after PCI error 16516 * and/or reset recovery has finished. 16517 **/ 16518static void i40e_pci_error_resume(struct pci_dev *pdev) 16519{ 16520 struct i40e_pf *pf = pci_get_drvdata(pdev); 16521 16522 dev_dbg(&pdev->dev, "%s\n", __func__); 16523 if (test_bit(__I40E_SUSPENDED, pf->state)) 16524 return; 16525 16526 i40e_io_resume(pf); 16527} 16528 16529/** 16530 * i40e_shutdown - PCI callback for shutting down 16531 * @pdev: PCI device information struct 16532 **/ 16533static void i40e_shutdown(struct pci_dev *pdev) 16534{ 16535 struct i40e_pf *pf = pci_get_drvdata(pdev); 16536 struct i40e_hw *hw = &pf->hw; 16537 16538 set_bit(__I40E_SUSPENDED, pf->state); 16539 set_bit(__I40E_DOWN, pf->state); 16540 16541 timer_delete_sync(&pf->service_timer); 16542 cancel_work_sync(&pf->service_task); 16543 i40e_cloud_filter_exit(pf); 16544 i40e_fdir_teardown(pf); 16545 16546 /* Client close must be called explicitly here because the timer 16547 * has been stopped. 16548 */ 16549 i40e_notify_client_of_netdev_close(pf, false); 16550 16551 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) && 16552 pf->wol_en) 16553 i40e_enable_mc_magic_wake(pf); 16554 16555 i40e_prep_for_reset(pf); 16556 16557 wr32(hw, I40E_PFPM_APM, 16558 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16559 wr32(hw, I40E_PFPM_WUFC, 16560 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16561 16562 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16563 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16564 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) 16565 free_irq(pf->pdev->irq, pf); 16566 16567 /* Since we're going to destroy queues during the 16568 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16569 * whole section 16570 */ 16571 rtnl_lock(); 16572 i40e_clear_interrupt_scheme(pf); 16573 rtnl_unlock(); 16574 16575 if (system_state == SYSTEM_POWER_OFF) { 16576 pci_wake_from_d3(pdev, pf->wol_en); 16577 pci_set_power_state(pdev, PCI_D3hot); 16578 } 16579} 16580 16581/** 16582 * i40e_suspend - PM callback for moving to D3 16583 * @dev: generic device information structure 16584 **/ 16585static int i40e_suspend(struct device *dev) 16586{ 16587 struct i40e_pf *pf = dev_get_drvdata(dev); 16588 16589 /* If we're already suspended, then there is nothing to do */ 16590 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 16591 return 0; 16592 return i40e_io_suspend(pf); 16593} 16594 16595/** 16596 * i40e_resume - PM callback for waking up from D3 16597 * @dev: generic device information structure 16598 **/ 16599static int i40e_resume(struct device *dev) 16600{ 16601 struct i40e_pf *pf = dev_get_drvdata(dev); 16602 16603 /* If we're not suspended, then there is nothing to do */ 16604 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16605 return 0; 16606 return i40e_io_resume(pf); 16607} 16608 16609static const struct pci_error_handlers i40e_err_handler = { 16610 .error_detected = i40e_pci_error_detected, 16611 .slot_reset = i40e_pci_error_slot_reset, 16612 .reset_prepare = i40e_pci_error_reset_prepare, 16613 .reset_done = i40e_pci_error_reset_done, 16614 .resume = i40e_pci_error_resume, 16615}; 16616 16617static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 16618 16619static struct pci_driver i40e_driver = { 16620 .name = i40e_driver_name, 16621 .id_table = i40e_pci_tbl, 16622 .probe = i40e_probe, 16623 .remove = i40e_remove, 16624 .driver.pm = pm_sleep_ptr(&i40e_pm_ops), 16625 .shutdown = i40e_shutdown, 16626 .err_handler = &i40e_err_handler, 16627 .sriov_configure = i40e_pci_sriov_configure, 16628}; 16629 16630/** 16631 * i40e_init_module - Driver registration routine 16632 * 16633 * i40e_init_module is the first routine called when the driver is 16634 * loaded. All it does is register with the PCI subsystem. 16635 **/ 16636static int __init i40e_init_module(void) 16637{ 16638 int err; 16639 16640 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 16641 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 16642 16643 /* There is no need to throttle the number of active tasks because 16644 * each device limits its own task using a state bit for scheduling 16645 * the service task, and the device tasks do not interfere with each 16646 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 16647 * since we need to be able to guarantee forward progress even under 16648 * memory pressure. 16649 */ 16650 i40e_wq = alloc_workqueue("%s", WQ_PERCPU, 0, i40e_driver_name); 16651 if (!i40e_wq) { 16652 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 16653 return -ENOMEM; 16654 } 16655 16656 i40e_dbg_init(); 16657 err = pci_register_driver(&i40e_driver); 16658 if (err) { 16659 destroy_workqueue(i40e_wq); 16660 i40e_dbg_exit(); 16661 return err; 16662 } 16663 16664 return 0; 16665} 16666module_init(i40e_init_module); 16667 16668/** 16669 * i40e_exit_module - Driver exit cleanup routine 16670 * 16671 * i40e_exit_module is called just before the driver is removed 16672 * from memory. 16673 **/ 16674static void __exit i40e_exit_module(void) 16675{ 16676 pci_unregister_driver(&i40e_driver); 16677 destroy_workqueue(i40e_wq); 16678 ida_destroy(&i40e_client_ida); 16679 i40e_dbg_exit(); 16680} 16681module_exit(i40e_exit_module);
~/
Read-only file preview.