FILE
cxgb4_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
1/* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37#include <linux/bitmap.h> 38#include <linux/crc32.h> 39#include <linux/ctype.h> 40#include <linux/debugfs.h> 41#include <linux/err.h> 42#include <linux/etherdevice.h> 43#include <linux/firmware.h> 44#include <linux/if.h> 45#include <linux/if_vlan.h> 46#include <linux/init.h> 47#include <linux/log2.h> 48#include <linux/mdio.h> 49#include <linux/module.h> 50#include <linux/moduleparam.h> 51#include <linux/mutex.h> 52#include <linux/netdevice.h> 53#include <linux/pci.h> 54#include <linux/rtnetlink.h> 55#include <linux/sched.h> 56#include <linux/seq_file.h> 57#include <linux/sockios.h> 58#include <linux/vmalloc.h> 59#include <linux/workqueue.h> 60#include <net/neighbour.h> 61#include <net/netevent.h> 62#include <net/addrconf.h> 63#include <net/bonding.h> 64#include <linux/uaccess.h> 65#include <linux/crash_dump.h> 66#include <net/udp_tunnel.h> 67#include <net/xfrm.h> 68#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) 69#include <net/tls.h> 70#endif 71 72#include "cxgb4.h" 73#include "cxgb4_filter.h" 74#include "t4_regs.h" 75#include "t4_values.h" 76#include "t4_msg.h" 77#include "t4fw_api.h" 78#include "t4fw_version.h" 79#include "cxgb4_dcb.h" 80#include "srq.h" 81#include "cxgb4_debugfs.h" 82#include "clip_tbl.h" 83#include "l2t.h" 84#include "smt.h" 85#include "sched.h" 86#include "cxgb4_tc_u32.h" 87#include "cxgb4_tc_flower.h" 88#include "cxgb4_tc_mqprio.h" 89#include "cxgb4_tc_matchall.h" 90#include "cxgb4_ptp.h" 91#include "cxgb4_cudbg.h" 92 93char cxgb4_driver_name[] = KBUILD_MODNAME; 94 95#define DRV_DESC "Chelsio T4/T5/T6 Network Driver" 96 97#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 98 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 99 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 100 101/* Macros needed to support the PCI Device ID Table ... 102 */ 103#define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 104 static const struct pci_device_id cxgb4_pci_tbl[] = { 105#define CXGB4_UNIFIED_PF 0x4 106 107#define CH_PCI_DEVICE_ID_FUNCTION CXGB4_UNIFIED_PF 108 109/* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 110 * called for both. 111 */ 112#define CH_PCI_DEVICE_ID_FUNCTION2 0x0 113 114#define CH_PCI_ID_TABLE_ENTRY(devid) \ 115 {PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF} 116 117#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 118 { 0, } \ 119 } 120 121#include "t4_pci_id_tbl.h" 122 123#define FW4_FNAME "cxgb4/t4fw.bin" 124#define FW5_FNAME "cxgb4/t5fw.bin" 125#define FW6_FNAME "cxgb4/t6fw.bin" 126#define FW4_CFNAME "cxgb4/t4-config.txt" 127#define FW5_CFNAME "cxgb4/t5-config.txt" 128#define FW6_CFNAME "cxgb4/t6-config.txt" 129#define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" 130#define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" 131#define PHY_AQ1202_DEVICEID 0x4409 132#define PHY_BCM84834_DEVICEID 0x4486 133 134MODULE_DESCRIPTION(DRV_DESC); 135MODULE_AUTHOR("Chelsio Communications"); 136MODULE_LICENSE("Dual BSD/GPL"); 137MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 138MODULE_FIRMWARE(FW4_FNAME); 139MODULE_FIRMWARE(FW5_FNAME); 140MODULE_FIRMWARE(FW6_FNAME); 141 142/* 143 * The driver uses the best interrupt scheme available on a platform in the 144 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 145 * of these schemes the driver may consider as follows: 146 * 147 * msi = 2: choose from among all three options 148 * msi = 1: only consider MSI and INTx interrupts 149 * msi = 0: force INTx interrupts 150 */ 151static int msi = 2; 152 153module_param(msi, int, 0644); 154MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 155 156/* 157 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 158 * offset by 2 bytes in order to have the IP headers line up on 4-byte 159 * boundaries. This is a requirement for many architectures which will throw 160 * a machine check fault if an attempt is made to access one of the 4-byte IP 161 * header fields on a non-4-byte boundary. And it's a major performance issue 162 * even on some architectures which allow it like some implementations of the 163 * x86 ISA. However, some architectures don't mind this and for some very 164 * edge-case performance sensitive applications (like forwarding large volumes 165 * of small packets), setting this DMA offset to 0 will decrease the number of 166 * PCI-E Bus transfers enough to measurably affect performance. 167 */ 168static int rx_dma_offset = 2; 169 170/* TX Queue select used to determine what algorithm to use for selecting TX 171 * queue. Select between the kernel provided function (select_queue=0) or user 172 * cxgb_select_queue function (select_queue=1) 173 * 174 * Default: select_queue=0 175 */ 176static int select_queue; 177module_param(select_queue, int, 0644); 178MODULE_PARM_DESC(select_queue, 179 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 180 181static struct dentry *cxgb4_debugfs_root; 182 183LIST_HEAD(adapter_list); 184DEFINE_MUTEX(uld_mutex); 185LIST_HEAD(uld_list); 186 187static int cfg_queues(struct adapter *adap); 188 189static void link_report(struct net_device *dev) 190{ 191 if (!netif_carrier_ok(dev)) 192 netdev_info(dev, "link down\n"); 193 else { 194 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 195 196 const char *s; 197 const struct port_info *p = netdev_priv(dev); 198 199 switch (p->link_cfg.speed) { 200 case 100: 201 s = "100Mbps"; 202 break; 203 case 1000: 204 s = "1Gbps"; 205 break; 206 case 10000: 207 s = "10Gbps"; 208 break; 209 case 25000: 210 s = "25Gbps"; 211 break; 212 case 40000: 213 s = "40Gbps"; 214 break; 215 case 50000: 216 s = "50Gbps"; 217 break; 218 case 100000: 219 s = "100Gbps"; 220 break; 221 default: 222 pr_info("%s: unsupported speed: %d\n", 223 dev->name, p->link_cfg.speed); 224 return; 225 } 226 227 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 228 fc[p->link_cfg.fc]); 229 } 230} 231 232#ifdef CONFIG_CHELSIO_T4_DCB 233/* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 234static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 235{ 236 struct port_info *pi = netdev_priv(dev); 237 struct adapter *adap = pi->adapter; 238 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 239 int i; 240 241 /* We use a simple mapping of Port TX Queue Index to DCB 242 * Priority when we're enabling DCB. 243 */ 244 for (i = 0; i < pi->nqsets; i++, txq++) { 245 u32 name, value; 246 int err; 247 248 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 249 FW_PARAMS_PARAM_X_V( 250 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 251 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 252 value = enable ? i : 0xffffffff; 253 254 /* Since we can be called while atomic (from "interrupt 255 * level") we need to issue the Set Parameters Commannd 256 * without sleeping (timeout < 0). 257 */ 258 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, 259 &name, &value, 260 -FW_CMD_MAX_TIMEOUT); 261 262 if (err) 263 dev_err(adap->pdev_dev, 264 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 265 enable ? "set" : "unset", pi->port_id, i, -err); 266 else 267 txq->dcb_prio = enable ? value : 0; 268 } 269} 270 271int cxgb4_dcb_enabled(const struct net_device *dev) 272{ 273 struct port_info *pi = netdev_priv(dev); 274 275 if (!pi->dcb.enabled) 276 return 0; 277 278 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 279 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 280} 281#endif /* CONFIG_CHELSIO_T4_DCB */ 282 283void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 284{ 285 struct net_device *dev = adapter->port[port_id]; 286 287 /* Skip changes from disabled ports. */ 288 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 289 if (link_stat) 290 netif_carrier_on(dev); 291 else { 292#ifdef CONFIG_CHELSIO_T4_DCB 293 if (cxgb4_dcb_enabled(dev)) { 294 cxgb4_dcb_reset(dev); 295 dcb_tx_queue_prio_enable(dev, false); 296 } 297#endif /* CONFIG_CHELSIO_T4_DCB */ 298 netif_carrier_off(dev); 299 } 300 301 link_report(dev); 302 } 303} 304 305void t4_os_portmod_changed(struct adapter *adap, int port_id) 306{ 307 static const char *mod_str[] = { 308 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 309 }; 310 311 struct net_device *dev = adap->port[port_id]; 312 struct port_info *pi = netdev_priv(dev); 313 314 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 315 netdev_info(dev, "port module unplugged\n"); 316 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 317 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 318 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 319 netdev_info(dev, "%s: unsupported port module inserted\n", 320 dev->name); 321 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 322 netdev_info(dev, "%s: unknown port module inserted\n", 323 dev->name); 324 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) 325 netdev_info(dev, "%s: transceiver module error\n", dev->name); 326 else 327 netdev_info(dev, "%s: unknown module type %d inserted\n", 328 dev->name, pi->mod_type); 329 330 /* If the interface is running, then we'll need any "sticky" Link 331 * Parameters redone with a new Transceiver Module. 332 */ 333 pi->link_cfg.redo_l1cfg = netif_running(dev); 334} 335 336int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 337module_param(dbfifo_int_thresh, int, 0644); 338MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 339 340/* 341 * usecs to sleep while draining the dbfifo 342 */ 343static int dbfifo_drain_delay = 1000; 344module_param(dbfifo_drain_delay, int, 0644); 345MODULE_PARM_DESC(dbfifo_drain_delay, 346 "usecs to sleep while draining the dbfifo"); 347 348static inline int cxgb4_set_addr_hash(struct port_info *pi) 349{ 350 struct adapter *adap = pi->adapter; 351 u64 vec = 0; 352 bool ucast = false; 353 struct hash_mac_addr *entry; 354 355 /* Calculate the hash vector for the updated list and program it */ 356 list_for_each_entry(entry, &adap->mac_hlist, list) { 357 ucast |= is_unicast_ether_addr(entry->addr); 358 vec |= (1ULL << hash_mac_addr(entry->addr)); 359 } 360 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, 361 vec, false); 362} 363 364static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) 365{ 366 struct port_info *pi = netdev_priv(netdev); 367 struct adapter *adap = pi->adapter; 368 int ret; 369 u64 mhash = 0; 370 u64 uhash = 0; 371 /* idx stores the index of allocated filters, 372 * its size should be modified based on the number of 373 * MAC addresses that we allocate filters for 374 */ 375 376 u16 idx[1] = {}; 377 bool free = false; 378 bool ucast = is_unicast_ether_addr(mac_addr); 379 const u8 *maclist[1] = {mac_addr}; 380 struct hash_mac_addr *new_entry; 381 382 ret = cxgb4_alloc_mac_filt(adap, pi->viid, free, 1, maclist, 383 idx, ucast ? &uhash : &mhash, false); 384 if (ret < 0) 385 goto out; 386 /* if hash != 0, then add the addr to hash addr list 387 * so on the end we will calculate the hash for the 388 * list and program it 389 */ 390 if (uhash || mhash) { 391 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); 392 if (!new_entry) 393 return -ENOMEM; 394 ether_addr_copy(new_entry->addr, mac_addr); 395 list_add_tail(&new_entry->list, &adap->mac_hlist); 396 ret = cxgb4_set_addr_hash(pi); 397 } 398out: 399 return ret < 0 ? ret : 0; 400} 401 402static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) 403{ 404 struct port_info *pi = netdev_priv(netdev); 405 struct adapter *adap = pi->adapter; 406 int ret; 407 const u8 *maclist[1] = {mac_addr}; 408 struct hash_mac_addr *entry, *tmp; 409 410 /* If the MAC address to be removed is in the hash addr 411 * list, delete it from the list and update hash vector 412 */ 413 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { 414 if (ether_addr_equal(entry->addr, mac_addr)) { 415 list_del(&entry->list); 416 kfree(entry); 417 return cxgb4_set_addr_hash(pi); 418 } 419 } 420 421 ret = cxgb4_free_mac_filt(adap, pi->viid, 1, maclist, false); 422 return ret < 0 ? -EINVAL : 0; 423} 424 425/* 426 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 427 * If @mtu is -1 it is left unchanged. 428 */ 429static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 430{ 431 struct port_info *pi = netdev_priv(dev); 432 struct adapter *adapter = pi->adapter; 433 434 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 435 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 436 437 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, pi->viid_mirror, 438 mtu, (dev->flags & IFF_PROMISC) ? 1 : 0, 439 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 440 sleep_ok); 441} 442 443/** 444 * cxgb4_change_mac - Update match filter for a MAC address. 445 * @pi: the port_info 446 * @viid: the VI id 447 * @tcam_idx: TCAM index of existing filter for old value of MAC address, 448 * or -1 449 * @addr: the new MAC address value 450 * @persist: whether a new MAC allocation should be persistent 451 * @smt_idx: the destination to store the new SMT index. 452 * 453 * Modifies an MPS filter and sets it to the new MAC address if 454 * @tcam_idx >= 0, or adds the MAC address to a new filter if 455 * @tcam_idx < 0. In the latter case the address is added persistently 456 * if @persist is %true. 457 * Addresses are programmed to hash region, if tcam runs out of entries. 458 * 459 */ 460int cxgb4_change_mac(struct port_info *pi, unsigned int viid, 461 int *tcam_idx, const u8 *addr, bool persist, 462 u8 *smt_idx) 463{ 464 struct adapter *adapter = pi->adapter; 465 struct hash_mac_addr *entry, *new_entry; 466 int ret; 467 468 ret = t4_change_mac(adapter, adapter->mbox, viid, 469 *tcam_idx, addr, persist, smt_idx); 470 /* We ran out of TCAM entries. try programming hash region. */ 471 if (ret == -ENOMEM) { 472 /* If the MAC address to be updated is in the hash addr 473 * list, update it from the list 474 */ 475 list_for_each_entry(entry, &adapter->mac_hlist, list) { 476 if (entry->iface_mac) { 477 ether_addr_copy(entry->addr, addr); 478 goto set_hash; 479 } 480 } 481 new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); 482 if (!new_entry) 483 return -ENOMEM; 484 ether_addr_copy(new_entry->addr, addr); 485 new_entry->iface_mac = true; 486 list_add_tail(&new_entry->list, &adapter->mac_hlist); 487set_hash: 488 ret = cxgb4_set_addr_hash(pi); 489 } else if (ret >= 0) { 490 *tcam_idx = ret; 491 ret = 0; 492 } 493 494 return ret; 495} 496 497/* 498 * link_start - enable a port 499 * @dev: the port to enable 500 * 501 * Performs the MAC and PHY actions needed to enable a port. 502 */ 503static int link_start(struct net_device *dev) 504{ 505 struct port_info *pi = netdev_priv(dev); 506 unsigned int mb = pi->adapter->mbox; 507 int ret; 508 509 /* 510 * We do not set address filters and promiscuity here, the stack does 511 * that step explicitly. 512 */ 513 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, pi->viid_mirror, 514 dev->mtu, -1, -1, -1, 515 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 516 if (ret == 0) 517 ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt, 518 dev->dev_addr, true, &pi->smt_idx); 519 if (ret == 0) 520 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, 521 &pi->link_cfg); 522 if (ret == 0) { 523 local_bh_disable(); 524 ret = t4_enable_pi_params(pi->adapter, mb, pi, true, 525 true, CXGB4_DCB_ENABLED); 526 local_bh_enable(); 527 } 528 529 return ret; 530} 531 532#ifdef CONFIG_CHELSIO_T4_DCB 533/* Handle a Data Center Bridging update message from the firmware. */ 534static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 535{ 536 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 537 struct net_device *dev = adap->port[adap->chan_map[port]]; 538 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 539 int new_dcb_enabled; 540 541 cxgb4_dcb_handle_fw_update(adap, pcmd); 542 new_dcb_enabled = cxgb4_dcb_enabled(dev); 543 544 /* If the DCB has become enabled or disabled on the port then we're 545 * going to need to set up/tear down DCB Priority parameters for the 546 * TX Queues associated with the port. 547 */ 548 if (new_dcb_enabled != old_dcb_enabled) 549 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 550} 551#endif /* CONFIG_CHELSIO_T4_DCB */ 552 553/* Response queue handler for the FW event queue. 554 */ 555static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 556 const struct pkt_gl *gl) 557{ 558 u8 opcode = ((const struct rss_header *)rsp)->opcode; 559 560 rsp++; /* skip RSS header */ 561 562 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 563 */ 564 if (unlikely(opcode == CPL_FW4_MSG && 565 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 566 rsp++; 567 opcode = ((const struct rss_header *)rsp)->opcode; 568 rsp++; 569 if (opcode != CPL_SGE_EGR_UPDATE) { 570 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 571 , opcode); 572 goto out; 573 } 574 } 575 576 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 577 const struct cpl_sge_egr_update *p = (void *)rsp; 578 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 579 struct sge_txq *txq; 580 581 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 582 txq->restarts++; 583 if (txq->q_type == CXGB4_TXQ_ETH) { 584 struct sge_eth_txq *eq; 585 586 eq = container_of(txq, struct sge_eth_txq, q); 587 t4_sge_eth_txq_egress_update(q->adap, eq, -1); 588 } else { 589 struct sge_uld_txq *oq; 590 591 oq = container_of(txq, struct sge_uld_txq, q); 592 tasklet_schedule(&oq->qresume_tsk); 593 } 594 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 595 const struct cpl_fw6_msg *p = (void *)rsp; 596 597#ifdef CONFIG_CHELSIO_T4_DCB 598 const struct fw_port_cmd *pcmd = (const void *)p->data; 599 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 600 unsigned int action = 601 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 602 603 if (cmd == FW_PORT_CMD && 604 (action == FW_PORT_ACTION_GET_PORT_INFO || 605 action == FW_PORT_ACTION_GET_PORT_INFO32)) { 606 int port = FW_PORT_CMD_PORTID_G( 607 be32_to_cpu(pcmd->op_to_portid)); 608 struct net_device *dev; 609 int dcbxdis, state_input; 610 611 dev = q->adap->port[q->adap->chan_map[port]]; 612 dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO 613 ? !!(pcmd->u.info.dcbxdis_pkd & FW_PORT_CMD_DCBXDIS_F) 614 : !!(be32_to_cpu(pcmd->u.info32.lstatus32_to_cbllen32) 615 & FW_PORT_CMD_DCBXDIS32_F)); 616 state_input = (dcbxdis 617 ? CXGB4_DCB_INPUT_FW_DISABLED 618 : CXGB4_DCB_INPUT_FW_ENABLED); 619 620 cxgb4_dcb_state_fsm(dev, state_input); 621 } 622 623 if (cmd == FW_PORT_CMD && 624 action == FW_PORT_ACTION_L2_DCB_CFG) 625 dcb_rpl(q->adap, pcmd); 626 else 627#endif 628 if (p->type == 0) 629 t4_handle_fw_rpl(q->adap, p->data); 630 } else if (opcode == CPL_L2T_WRITE_RPL) { 631 const struct cpl_l2t_write_rpl *p = (void *)rsp; 632 633 do_l2t_write_rpl(q->adap, p); 634 } else if (opcode == CPL_SMT_WRITE_RPL) { 635 const struct cpl_smt_write_rpl *p = (void *)rsp; 636 637 do_smt_write_rpl(q->adap, p); 638 } else if (opcode == CPL_SET_TCB_RPL) { 639 const struct cpl_set_tcb_rpl *p = (void *)rsp; 640 641 filter_rpl(q->adap, p); 642 } else if (opcode == CPL_ACT_OPEN_RPL) { 643 const struct cpl_act_open_rpl *p = (void *)rsp; 644 645 hash_filter_rpl(q->adap, p); 646 } else if (opcode == CPL_ABORT_RPL_RSS) { 647 const struct cpl_abort_rpl_rss *p = (void *)rsp; 648 649 hash_del_filter_rpl(q->adap, p); 650 } else if (opcode == CPL_SRQ_TABLE_RPL) { 651 const struct cpl_srq_table_rpl *p = (void *)rsp; 652 653 do_srq_table_rpl(q->adap, p); 654 } else 655 dev_err(q->adap->pdev_dev, 656 "unexpected CPL %#x on FW event queue\n", opcode); 657out: 658 return 0; 659} 660 661static void disable_msi(struct adapter *adapter) 662{ 663 if (adapter->flags & CXGB4_USING_MSIX) { 664 pci_disable_msix(adapter->pdev); 665 adapter->flags &= ~CXGB4_USING_MSIX; 666 } else if (adapter->flags & CXGB4_USING_MSI) { 667 pci_disable_msi(adapter->pdev); 668 adapter->flags &= ~CXGB4_USING_MSI; 669 } 670} 671 672/* 673 * Interrupt handler for non-data events used with MSI-X. 674 */ 675static irqreturn_t t4_nondata_intr(int irq, void *cookie) 676{ 677 struct adapter *adap = cookie; 678 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 679 680 if (v & PFSW_F) { 681 adap->swintr = 1; 682 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 683 } 684 if (adap->flags & CXGB4_MASTER_PF) 685 t4_slow_intr_handler(adap); 686 return IRQ_HANDLED; 687} 688 689int cxgb4_set_msix_aff(struct adapter *adap, unsigned short vec, 690 cpumask_var_t *aff_mask, int idx) 691{ 692 int rv; 693 694 if (!zalloc_cpumask_var(aff_mask, GFP_KERNEL)) { 695 dev_err(adap->pdev_dev, "alloc_cpumask_var failed\n"); 696 return -ENOMEM; 697 } 698 699 cpumask_set_cpu(cpumask_local_spread(idx, dev_to_node(adap->pdev_dev)), 700 *aff_mask); 701 702 rv = irq_set_affinity_hint(vec, *aff_mask); 703 if (rv) 704 dev_warn(adap->pdev_dev, 705 "irq_set_affinity_hint %u failed %d\n", 706 vec, rv); 707 708 return 0; 709} 710 711void cxgb4_clear_msix_aff(unsigned short vec, cpumask_var_t aff_mask) 712{ 713 irq_set_affinity_hint(vec, NULL); 714 free_cpumask_var(aff_mask); 715} 716 717static int request_msix_queue_irqs(struct adapter *adap) 718{ 719 struct sge *s = &adap->sge; 720 struct msix_info *minfo; 721 int err, ethqidx; 722 723 if (s->fwevtq_msix_idx < 0) 724 return -ENOMEM; 725 726 err = request_irq(adap->msix_info[s->fwevtq_msix_idx].vec, 727 t4_sge_intr_msix, 0, 728 adap->msix_info[s->fwevtq_msix_idx].desc, 729 &s->fw_evtq); 730 if (err) 731 return err; 732 733 for_each_ethrxq(s, ethqidx) { 734 minfo = s->ethrxq[ethqidx].msix; 735 err = request_irq(minfo->vec, 736 t4_sge_intr_msix, 0, 737 minfo->desc, 738 &s->ethrxq[ethqidx].rspq); 739 if (err) 740 goto unwind; 741 742 cxgb4_set_msix_aff(adap, minfo->vec, 743 &minfo->aff_mask, ethqidx); 744 } 745 return 0; 746 747unwind: 748 while (--ethqidx >= 0) { 749 minfo = s->ethrxq[ethqidx].msix; 750 cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); 751 free_irq(minfo->vec, &s->ethrxq[ethqidx].rspq); 752 } 753 free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq); 754 return err; 755} 756 757static void free_msix_queue_irqs(struct adapter *adap) 758{ 759 struct sge *s = &adap->sge; 760 struct msix_info *minfo; 761 int i; 762 763 free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq); 764 for_each_ethrxq(s, i) { 765 minfo = s->ethrxq[i].msix; 766 cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask); 767 free_irq(minfo->vec, &s->ethrxq[i].rspq); 768 } 769} 770 771static int setup_ppod_edram(struct adapter *adap) 772{ 773 unsigned int param, val; 774 int ret; 775 776 /* Driver sends FW_PARAMS_PARAM_DEV_PPOD_EDRAM read command to check 777 * if firmware supports ppod edram feature or not. If firmware 778 * returns 1, then driver can enable this feature by sending 779 * FW_PARAMS_PARAM_DEV_PPOD_EDRAM write command with value 1 to 780 * enable ppod edram feature. 781 */ 782 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 783 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PPOD_EDRAM)); 784 785 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); 786 if (ret < 0) { 787 dev_warn(adap->pdev_dev, 788 "querying PPOD_EDRAM support failed: %d\n", 789 ret); 790 return -1; 791 } 792 793 if (val != 1) 794 return -1; 795 796 ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, ¶m, &val); 797 if (ret < 0) { 798 dev_err(adap->pdev_dev, 799 "setting PPOD_EDRAM failed: %d\n", ret); 800 return -1; 801 } 802 return 0; 803} 804 805static void adap_config_hpfilter(struct adapter *adapter) 806{ 807 u32 param, val = 0; 808 int ret; 809 810 /* Enable HP filter region. Older fw will fail this request and 811 * it is fine. 812 */ 813 param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT); 814 ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0, 815 1, ¶m, &val); 816 817 /* An error means FW doesn't know about HP filter support, 818 * it's not a problem, don't return an error. 819 */ 820 if (ret < 0) 821 dev_err(adapter->pdev_dev, 822 "HP filter region isn't supported by FW\n"); 823} 824 825static int cxgb4_config_rss(const struct port_info *pi, u16 *rss, 826 u16 rss_size, u16 viid) 827{ 828 struct adapter *adap = pi->adapter; 829 int ret; 830 831 ret = t4_config_rss_range(adap, adap->mbox, viid, 0, rss_size, rss, 832 rss_size); 833 if (ret) 834 return ret; 835 836 /* If Tunnel All Lookup isn't specified in the global RSS 837 * Configuration, then we need to specify a default Ingress 838 * Queue for any ingress packets which aren't hashed. We'll 839 * use our first ingress queue ... 840 */ 841 return t4_config_vi_rss(adap, adap->mbox, viid, 842 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | 843 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | 844 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | 845 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | 846 FW_RSS_VI_CONFIG_CMD_UDPEN_F, 847 rss[0]); 848} 849 850/** 851 * cxgb4_write_rss - write the RSS table for a given port 852 * @pi: the port 853 * @queues: array of queue indices for RSS 854 * 855 * Sets up the portion of the HW RSS table for the port's VI to distribute 856 * packets to the Rx queues in @queues. 857 * Should never be called before setting up sge eth rx queues 858 */ 859int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) 860{ 861 struct adapter *adapter = pi->adapter; 862 const struct sge_eth_rxq *rxq; 863 int i, err; 864 u16 *rss; 865 866 rxq = &adapter->sge.ethrxq[pi->first_qset]; 867 rss = kmalloc_array(pi->rss_size, sizeof(u16), GFP_KERNEL); 868 if (!rss) 869 return -ENOMEM; 870 871 /* map the queue indices to queue ids */ 872 for (i = 0; i < pi->rss_size; i++, queues++) 873 rss[i] = rxq[*queues].rspq.abs_id; 874 875 err = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid); 876 kfree(rss); 877 return err; 878} 879 880/** 881 * setup_rss - configure RSS 882 * @adap: the adapter 883 * 884 * Sets up RSS for each port. 885 */ 886static int setup_rss(struct adapter *adap) 887{ 888 int i, j, err; 889 890 for_each_port(adap, i) { 891 const struct port_info *pi = adap2pinfo(adap, i); 892 893 /* Fill default values with equal distribution */ 894 for (j = 0; j < pi->rss_size; j++) 895 pi->rss[j] = j % pi->nqsets; 896 897 err = cxgb4_write_rss(pi, pi->rss); 898 if (err) 899 return err; 900 } 901 return 0; 902} 903 904/* 905 * Return the channel of the ingress queue with the given qid. 906 */ 907static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 908{ 909 qid -= p->ingr_start; 910 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 911} 912 913void cxgb4_quiesce_rx(struct sge_rspq *q) 914{ 915 if (q->handler) 916 napi_disable(&q->napi); 917} 918 919/* 920 * Wait until all NAPI handlers are descheduled. 921 */ 922static void quiesce_rx(struct adapter *adap) 923{ 924 int i; 925 926 for (i = 0; i < adap->sge.ingr_sz; i++) { 927 struct sge_rspq *q = adap->sge.ingr_map[i]; 928 929 if (!q) 930 continue; 931 932 cxgb4_quiesce_rx(q); 933 } 934} 935 936/* Disable interrupt and napi handler */ 937static void disable_interrupts(struct adapter *adap) 938{ 939 struct sge *s = &adap->sge; 940 941 if (adap->flags & CXGB4_FULL_INIT_DONE) { 942 t4_intr_disable(adap); 943 if (adap->flags & CXGB4_USING_MSIX) { 944 free_msix_queue_irqs(adap); 945 free_irq(adap->msix_info[s->nd_msix_idx].vec, 946 adap); 947 } else { 948 free_irq(adap->pdev->irq, adap); 949 } 950 quiesce_rx(adap); 951 } 952} 953 954void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q) 955{ 956 if (q->handler) 957 napi_enable(&q->napi); 958 959 /* 0-increment GTS to start the timer and enable interrupts */ 960 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 961 SEINTARM_V(q->intr_params) | 962 INGRESSQID_V(q->cntxt_id)); 963} 964 965/* 966 * Enable NAPI scheduling and interrupt generation for all Rx queues. 967 */ 968static void enable_rx(struct adapter *adap) 969{ 970 int i; 971 972 for (i = 0; i < adap->sge.ingr_sz; i++) { 973 struct sge_rspq *q = adap->sge.ingr_map[i]; 974 975 if (!q) 976 continue; 977 978 cxgb4_enable_rx(adap, q); 979 } 980} 981 982static int setup_non_data_intr(struct adapter *adap) 983{ 984 int msix; 985 986 adap->sge.nd_msix_idx = -1; 987 if (!(adap->flags & CXGB4_USING_MSIX)) 988 return 0; 989 990 /* Request MSI-X vector for non-data interrupt */ 991 msix = cxgb4_get_msix_idx_from_bmap(adap); 992 if (msix < 0) 993 return -ENOMEM; 994 995 snprintf(adap->msix_info[msix].desc, 996 sizeof(adap->msix_info[msix].desc), 997 "%s", adap->port[0]->name); 998 999 adap->sge.nd_msix_idx = msix; 1000 return 0; 1001} 1002 1003static int setup_fw_sge_queues(struct adapter *adap) 1004{ 1005 struct sge *s = &adap->sge; 1006 int msix, err = 0; 1007 1008 bitmap_zero(s->starving_fl, s->egr_sz); 1009 bitmap_zero(s->txq_maperr, s->egr_sz); 1010 1011 if (adap->flags & CXGB4_USING_MSIX) { 1012 s->fwevtq_msix_idx = -1; 1013 msix = cxgb4_get_msix_idx_from_bmap(adap); 1014 if (msix < 0) 1015 return -ENOMEM; 1016 1017 snprintf(adap->msix_info[msix].desc, 1018 sizeof(adap->msix_info[msix].desc), 1019 "%s-FWeventq", adap->port[0]->name); 1020 } else { 1021 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 1022 NULL, NULL, NULL, -1); 1023 if (err) 1024 return err; 1025 msix = -((int)s->intrq.abs_id + 1); 1026 } 1027 1028 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 1029 msix, NULL, fwevtq_handler, NULL, -1); 1030 if (err && msix >= 0) 1031 cxgb4_free_msix_idx_in_bmap(adap, msix); 1032 1033 s->fwevtq_msix_idx = msix; 1034 return err; 1035} 1036 1037/** 1038 * setup_sge_queues - configure SGE Tx/Rx/response queues 1039 * @adap: the adapter 1040 * 1041 * Determines how many sets of SGE queues to use and initializes them. 1042 * We support multiple queue sets per port if we have MSI-X, otherwise 1043 * just one queue set per port. 1044 */ 1045static int setup_sge_queues(struct adapter *adap) 1046{ 1047 struct sge_uld_rxq_info *rxq_info = NULL; 1048 struct sge *s = &adap->sge; 1049 unsigned int cmplqid = 0; 1050 int err, i, j, msix = 0; 1051 1052 if (is_uld(adap)) 1053 rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; 1054 1055 if (!(adap->flags & CXGB4_USING_MSIX)) 1056 msix = -((int)s->intrq.abs_id + 1); 1057 1058 for_each_port(adap, i) { 1059 struct net_device *dev = adap->port[i]; 1060 struct port_info *pi = netdev_priv(dev); 1061 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 1062 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 1063 1064 for (j = 0; j < pi->nqsets; j++, q++) { 1065 if (msix >= 0) { 1066 msix = cxgb4_get_msix_idx_from_bmap(adap); 1067 if (msix < 0) { 1068 err = msix; 1069 goto freeout; 1070 } 1071 1072 snprintf(adap->msix_info[msix].desc, 1073 sizeof(adap->msix_info[msix].desc), 1074 "%s-Rx%d", dev->name, j); 1075 q->msix = &adap->msix_info[msix]; 1076 } 1077 1078 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 1079 msix, &q->fl, 1080 t4_ethrx_handler, 1081 NULL, 1082 t4_get_tp_ch_map(adap, 1083 pi->tx_chan)); 1084 if (err) 1085 goto freeout; 1086 q->rspq.idx = j; 1087 memset(&q->stats, 0, sizeof(q->stats)); 1088 } 1089 1090 q = &s->ethrxq[pi->first_qset]; 1091 for (j = 0; j < pi->nqsets; j++, t++, q++) { 1092 err = t4_sge_alloc_eth_txq(adap, t, dev, 1093 netdev_get_tx_queue(dev, j), 1094 q->rspq.cntxt_id, 1095 !!(adap->flags & CXGB4_SGE_DBQ_TIMER)); 1096 if (err) 1097 goto freeout; 1098 } 1099 } 1100 1101 for_each_port(adap, i) { 1102 /* Note that cmplqid below is 0 if we don't 1103 * have RDMA queues, and that's the right value. 1104 */ 1105 if (rxq_info) 1106 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; 1107 1108 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 1109 s->fw_evtq.cntxt_id, cmplqid); 1110 if (err) 1111 goto freeout; 1112 } 1113 1114 if (!is_t4(adap->params.chip)) { 1115 err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0], 1116 netdev_get_tx_queue(adap->port[0], 0) 1117 , s->fw_evtq.cntxt_id, false); 1118 if (err) 1119 goto freeout; 1120 } 1121 1122 t4_write_reg(adap, is_t4(adap->params.chip) ? 1123 MPS_TRC_RSS_CONTROL_A : 1124 MPS_T5_TRC_RSS_CONTROL_A, 1125 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 1126 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 1127 return 0; 1128freeout: 1129 dev_err(adap->pdev_dev, "Can't allocate queues, err=%d\n", -err); 1130 t4_free_sge_resources(adap); 1131 return err; 1132} 1133 1134static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 1135 struct net_device *sb_dev) 1136{ 1137 int txq; 1138 1139#ifdef CONFIG_CHELSIO_T4_DCB 1140 /* If a Data Center Bridging has been successfully negotiated on this 1141 * link then we'll use the skb's priority to map it to a TX Queue. 1142 * The skb's priority is determined via the VLAN Tag Priority Code 1143 * Point field. 1144 */ 1145 if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) { 1146 u16 vlan_tci; 1147 int err; 1148 1149 err = vlan_get_tag(skb, &vlan_tci); 1150 if (unlikely(err)) { 1151 if (net_ratelimit()) 1152 netdev_warn(dev, 1153 "TX Packet without VLAN Tag on DCB Link\n"); 1154 txq = 0; 1155 } else { 1156 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1157#ifdef CONFIG_CHELSIO_T4_FCOE 1158 if (skb->protocol == htons(ETH_P_FCOE)) 1159 txq = skb->priority & 0x7; 1160#endif /* CONFIG_CHELSIO_T4_FCOE */ 1161 } 1162 return txq; 1163 } 1164#endif /* CONFIG_CHELSIO_T4_DCB */ 1165 1166 if (dev->num_tc) { 1167 struct port_info *pi = netdev2pinfo(dev); 1168 u8 ver, proto; 1169 1170 ver = ip_hdr(skb)->version; 1171 proto = (ver == 6) ? ipv6_hdr(skb)->nexthdr : 1172 ip_hdr(skb)->protocol; 1173 1174 /* Send unsupported traffic pattern to normal NIC queues. */ 1175 txq = netdev_pick_tx(dev, skb, sb_dev); 1176 if (xfrm_offload(skb) || is_ptp_enabled(skb, dev) || 1177 skb->encapsulation || 1178 tls_is_skb_tx_device_offloaded(skb) || 1179 (proto != IPPROTO_TCP && proto != IPPROTO_UDP)) 1180 txq = txq % pi->nqsets; 1181 1182 return txq; 1183 } 1184 1185 if (select_queue) { 1186 txq = (skb_rx_queue_recorded(skb) 1187 ? skb_get_rx_queue(skb) 1188 : smp_processor_id()); 1189 1190 while (unlikely(txq >= dev->real_num_tx_queues)) 1191 txq -= dev->real_num_tx_queues; 1192 1193 return txq; 1194 } 1195 1196 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 1197} 1198 1199static int closest_timer(const struct sge *s, int time) 1200{ 1201 int i, delta, match = 0, min_delta = INT_MAX; 1202 1203 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 1204 delta = time - s->timer_val[i]; 1205 if (delta < 0) 1206 delta = -delta; 1207 if (delta < min_delta) { 1208 min_delta = delta; 1209 match = i; 1210 } 1211 } 1212 return match; 1213} 1214 1215static int closest_thres(const struct sge *s, int thres) 1216{ 1217 int i, delta, match = 0, min_delta = INT_MAX; 1218 1219 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 1220 delta = thres - s->counter_val[i]; 1221 if (delta < 0) 1222 delta = -delta; 1223 if (delta < min_delta) { 1224 min_delta = delta; 1225 match = i; 1226 } 1227 } 1228 return match; 1229} 1230 1231/** 1232 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters 1233 * @q: the Rx queue 1234 * @us: the hold-off time in us, or 0 to disable timer 1235 * @cnt: the hold-off packet count, or 0 to disable counter 1236 * 1237 * Sets an Rx queue's interrupt hold-off time and packet count. At least 1238 * one of the two needs to be enabled for the queue to generate interrupts. 1239 */ 1240int cxgb4_set_rspq_intr_params(struct sge_rspq *q, 1241 unsigned int us, unsigned int cnt) 1242{ 1243 struct adapter *adap = q->adap; 1244 1245 if ((us | cnt) == 0) 1246 cnt = 1; 1247 1248 if (cnt) { 1249 int err; 1250 u32 v, new_idx; 1251 1252 new_idx = closest_thres(&adap->sge, cnt); 1253 if (q->desc && q->pktcnt_idx != new_idx) { 1254 /* the queue has already been created, update it */ 1255 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 1256 FW_PARAMS_PARAM_X_V( 1257 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 1258 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 1259 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 1260 &v, &new_idx); 1261 if (err) 1262 return err; 1263 } 1264 q->pktcnt_idx = new_idx; 1265 } 1266 1267 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 1268 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); 1269 return 0; 1270} 1271 1272static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 1273{ 1274 netdev_features_t changed = dev->features ^ features; 1275 const struct port_info *pi = netdev_priv(dev); 1276 int err; 1277 1278 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 1279 return 0; 1280 1281 err = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid, 1282 pi->viid_mirror, -1, -1, -1, -1, 1283 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 1284 if (unlikely(err)) 1285 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 1286 return err; 1287} 1288 1289static int setup_debugfs(struct adapter *adap) 1290{ 1291 if (IS_ERR_OR_NULL(adap->debugfs_root)) 1292 return -1; 1293 1294#ifdef CONFIG_DEBUG_FS 1295 t4_setup_debugfs(adap); 1296#endif 1297 return 0; 1298} 1299 1300static void cxgb4_port_mirror_free_rxq(struct adapter *adap, 1301 struct sge_eth_rxq *mirror_rxq) 1302{ 1303 if ((adap->flags & CXGB4_FULL_INIT_DONE) && 1304 !(adap->flags & CXGB4_SHUTTING_DOWN)) 1305 cxgb4_quiesce_rx(&mirror_rxq->rspq); 1306 1307 if (adap->flags & CXGB4_USING_MSIX) { 1308 cxgb4_clear_msix_aff(mirror_rxq->msix->vec, 1309 mirror_rxq->msix->aff_mask); 1310 free_irq(mirror_rxq->msix->vec, &mirror_rxq->rspq); 1311 cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx); 1312 } 1313 1314 free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl); 1315} 1316 1317static int cxgb4_port_mirror_alloc_queues(struct net_device *dev) 1318{ 1319 struct port_info *pi = netdev2pinfo(dev); 1320 struct adapter *adap = netdev2adap(dev); 1321 struct sge_eth_rxq *mirror_rxq; 1322 struct sge *s = &adap->sge; 1323 int ret = 0, msix = 0; 1324 u16 i, rxqid; 1325 u16 *rss; 1326 1327 if (!pi->vi_mirror_count) 1328 return 0; 1329 1330 if (s->mirror_rxq[pi->port_id]) 1331 return 0; 1332 1333 mirror_rxq = kcalloc(pi->nmirrorqsets, sizeof(*mirror_rxq), GFP_KERNEL); 1334 if (!mirror_rxq) 1335 return -ENOMEM; 1336 1337 s->mirror_rxq[pi->port_id] = mirror_rxq; 1338 1339 if (!(adap->flags & CXGB4_USING_MSIX)) 1340 msix = -((int)adap->sge.intrq.abs_id + 1); 1341 1342 for (i = 0, rxqid = 0; i < pi->nmirrorqsets; i++, rxqid++) { 1343 mirror_rxq = &s->mirror_rxq[pi->port_id][i]; 1344 1345 /* Allocate Mirror Rxqs */ 1346 if (msix >= 0) { 1347 msix = cxgb4_get_msix_idx_from_bmap(adap); 1348 if (msix < 0) { 1349 ret = msix; 1350 goto out_free_queues; 1351 } 1352 1353 mirror_rxq->msix = &adap->msix_info[msix]; 1354 snprintf(mirror_rxq->msix->desc, 1355 sizeof(mirror_rxq->msix->desc), 1356 "%s-mirrorrxq%d", dev->name, i); 1357 } 1358 1359 init_rspq(adap, &mirror_rxq->rspq, 1360 CXGB4_MIRROR_RXQ_DEFAULT_INTR_USEC, 1361 CXGB4_MIRROR_RXQ_DEFAULT_PKT_CNT, 1362 CXGB4_MIRROR_RXQ_DEFAULT_DESC_NUM, 1363 CXGB4_MIRROR_RXQ_DEFAULT_DESC_SIZE); 1364 1365 mirror_rxq->fl.size = CXGB4_MIRROR_FLQ_DEFAULT_DESC_NUM; 1366 1367 ret = t4_sge_alloc_rxq(adap, &mirror_rxq->rspq, false, 1368 dev, msix, &mirror_rxq->fl, 1369 t4_ethrx_handler, NULL, 0); 1370 if (ret) 1371 goto out_free_msix_idx; 1372 1373 /* Setup MSI-X vectors for Mirror Rxqs */ 1374 if (adap->flags & CXGB4_USING_MSIX) { 1375 ret = request_irq(mirror_rxq->msix->vec, 1376 t4_sge_intr_msix, 0, 1377 mirror_rxq->msix->desc, 1378 &mirror_rxq->rspq); 1379 if (ret) 1380 goto out_free_rxq; 1381 1382 cxgb4_set_msix_aff(adap, mirror_rxq->msix->vec, 1383 &mirror_rxq->msix->aff_mask, i); 1384 } 1385 1386 /* Start NAPI for Mirror Rxqs */ 1387 cxgb4_enable_rx(adap, &mirror_rxq->rspq); 1388 } 1389 1390 /* Setup RSS for Mirror Rxqs */ 1391 rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 1392 if (!rss) { 1393 ret = -ENOMEM; 1394 goto out_free_queues; 1395 } 1396 1397 mirror_rxq = &s->mirror_rxq[pi->port_id][0]; 1398 for (i = 0; i < pi->rss_size; i++) 1399 rss[i] = mirror_rxq[i % pi->nmirrorqsets].rspq.abs_id; 1400 1401 ret = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid_mirror); 1402 kfree(rss); 1403 if (ret) 1404 goto out_free_queues; 1405 1406 return 0; 1407 1408out_free_rxq: 1409 free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl); 1410 1411out_free_msix_idx: 1412 cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx); 1413 1414out_free_queues: 1415 while (rxqid-- > 0) 1416 cxgb4_port_mirror_free_rxq(adap, 1417 &s->mirror_rxq[pi->port_id][rxqid]); 1418 1419 kfree(s->mirror_rxq[pi->port_id]); 1420 s->mirror_rxq[pi->port_id] = NULL; 1421 return ret; 1422} 1423 1424static void cxgb4_port_mirror_free_queues(struct net_device *dev) 1425{ 1426 struct port_info *pi = netdev2pinfo(dev); 1427 struct adapter *adap = netdev2adap(dev); 1428 struct sge *s = &adap->sge; 1429 u16 i; 1430 1431 if (!pi->vi_mirror_count) 1432 return; 1433 1434 if (!s->mirror_rxq[pi->port_id]) 1435 return; 1436 1437 for (i = 0; i < pi->nmirrorqsets; i++) 1438 cxgb4_port_mirror_free_rxq(adap, 1439 &s->mirror_rxq[pi->port_id][i]); 1440 1441 kfree(s->mirror_rxq[pi->port_id]); 1442 s->mirror_rxq[pi->port_id] = NULL; 1443} 1444 1445static int cxgb4_port_mirror_start(struct net_device *dev) 1446{ 1447 struct port_info *pi = netdev2pinfo(dev); 1448 struct adapter *adap = netdev2adap(dev); 1449 int ret, idx = -1; 1450 1451 if (!pi->vi_mirror_count) 1452 return 0; 1453 1454 /* Mirror VIs can be created dynamically after stack had 1455 * already setup Rx modes like MTU, promisc, allmulti, etc. 1456 * on main VI. So, parse what the stack had setup on the 1457 * main VI and update the same on the mirror VI. 1458 */ 1459 ret = t4_set_rxmode(adap, adap->mbox, pi->viid, pi->viid_mirror, 1460 dev->mtu, (dev->flags & IFF_PROMISC) ? 1 : 0, 1461 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, 1462 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 1463 if (ret) { 1464 dev_err(adap->pdev_dev, 1465 "Failed start up Rx mode for Mirror VI 0x%x, ret: %d\n", 1466 pi->viid_mirror, ret); 1467 return ret; 1468 } 1469 1470 /* Enable replication bit for the device's MAC address 1471 * in MPS TCAM, so that the packets for the main VI are 1472 * replicated to mirror VI. 1473 */ 1474 ret = cxgb4_update_mac_filt(pi, pi->viid_mirror, &idx, 1475 dev->dev_addr, true, NULL); 1476 if (ret) { 1477 dev_err(adap->pdev_dev, 1478 "Failed updating MAC filter for Mirror VI 0x%x, ret: %d\n", 1479 pi->viid_mirror, ret); 1480 return ret; 1481 } 1482 1483 /* Enabling a Virtual Interface can result in an interrupt 1484 * during the processing of the VI Enable command and, in some 1485 * paths, result in an attempt to issue another command in the 1486 * interrupt context. Thus, we disable interrupts during the 1487 * course of the VI Enable command ... 1488 */ 1489 local_bh_disable(); 1490 ret = t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, true, true, 1491 false); 1492 local_bh_enable(); 1493 if (ret) 1494 dev_err(adap->pdev_dev, 1495 "Failed starting Mirror VI 0x%x, ret: %d\n", 1496 pi->viid_mirror, ret); 1497 1498 return ret; 1499} 1500 1501static void cxgb4_port_mirror_stop(struct net_device *dev) 1502{ 1503 struct port_info *pi = netdev2pinfo(dev); 1504 struct adapter *adap = netdev2adap(dev); 1505 1506 if (!pi->vi_mirror_count) 1507 return; 1508 1509 t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, false, false, 1510 false); 1511} 1512 1513int cxgb4_port_mirror_alloc(struct net_device *dev) 1514{ 1515 struct port_info *pi = netdev2pinfo(dev); 1516 struct adapter *adap = netdev2adap(dev); 1517 int ret = 0; 1518 1519 if (!pi->nmirrorqsets) 1520 return -EOPNOTSUPP; 1521 1522 mutex_lock(&pi->vi_mirror_mutex); 1523 if (pi->viid_mirror) { 1524 pi->vi_mirror_count++; 1525 goto out_unlock; 1526 } 1527 1528 ret = t4_init_port_mirror(pi, adap->mbox, pi->port_id, adap->pf, 0, 1529 &pi->viid_mirror); 1530 if (ret) 1531 goto out_unlock; 1532 1533 pi->vi_mirror_count = 1; 1534 1535 if (adap->flags & CXGB4_FULL_INIT_DONE) { 1536 ret = cxgb4_port_mirror_alloc_queues(dev); 1537 if (ret) 1538 goto out_free_vi; 1539 1540 ret = cxgb4_port_mirror_start(dev); 1541 if (ret) 1542 goto out_free_queues; 1543 } 1544 1545 mutex_unlock(&pi->vi_mirror_mutex); 1546 return 0; 1547 1548out_free_queues: 1549 cxgb4_port_mirror_free_queues(dev); 1550 1551out_free_vi: 1552 pi->vi_mirror_count = 0; 1553 t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror); 1554 pi->viid_mirror = 0; 1555 1556out_unlock: 1557 mutex_unlock(&pi->vi_mirror_mutex); 1558 return ret; 1559} 1560 1561void cxgb4_port_mirror_free(struct net_device *dev) 1562{ 1563 struct port_info *pi = netdev2pinfo(dev); 1564 struct adapter *adap = netdev2adap(dev); 1565 1566 mutex_lock(&pi->vi_mirror_mutex); 1567 if (!pi->viid_mirror) 1568 goto out_unlock; 1569 1570 if (pi->vi_mirror_count > 1) { 1571 pi->vi_mirror_count--; 1572 goto out_unlock; 1573 } 1574 1575 cxgb4_port_mirror_stop(dev); 1576 cxgb4_port_mirror_free_queues(dev); 1577 1578 pi->vi_mirror_count = 0; 1579 t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror); 1580 pi->viid_mirror = 0; 1581 1582out_unlock: 1583 mutex_unlock(&pi->vi_mirror_mutex); 1584} 1585 1586/* 1587 * upper-layer driver support 1588 */ 1589 1590/* 1591 * Allocate an active-open TID and set it to the supplied value. 1592 */ 1593int cxgb4_alloc_atid(struct tid_info *t, void *data) 1594{ 1595 int atid = -1; 1596 1597 spin_lock_bh(&t->atid_lock); 1598 if (t->afree) { 1599 union aopen_entry *p = t->afree; 1600 1601 atid = (p - t->atid_tab) + t->atid_base; 1602 t->afree = p->next; 1603 p->data = data; 1604 t->atids_in_use++; 1605 } 1606 spin_unlock_bh(&t->atid_lock); 1607 return atid; 1608} 1609EXPORT_SYMBOL(cxgb4_alloc_atid); 1610 1611/* 1612 * Release an active-open TID. 1613 */ 1614void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 1615{ 1616 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 1617 1618 spin_lock_bh(&t->atid_lock); 1619 p->next = t->afree; 1620 t->afree = p; 1621 t->atids_in_use--; 1622 spin_unlock_bh(&t->atid_lock); 1623} 1624EXPORT_SYMBOL(cxgb4_free_atid); 1625 1626/* 1627 * Allocate a server TID and set it to the supplied value. 1628 */ 1629int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 1630{ 1631 int stid; 1632 1633 spin_lock_bh(&t->stid_lock); 1634 if (family == PF_INET) { 1635 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 1636 if (stid < t->nstids) 1637 __set_bit(stid, t->stid_bmap); 1638 else 1639 stid = -1; 1640 } else { 1641 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); 1642 if (stid < 0) 1643 stid = -1; 1644 } 1645 if (stid >= 0) { 1646 t->stid_tab[stid].data = data; 1647 stid += t->stid_base; 1648 /* IPv6 requires max of 520 bits or 16 cells in TCAM 1649 * This is equivalent to 4 TIDs. With CLIP enabled it 1650 * needs 2 TIDs. 1651 */ 1652 if (family == PF_INET6) { 1653 t->stids_in_use += 2; 1654 t->v6_stids_in_use += 2; 1655 } else { 1656 t->stids_in_use++; 1657 } 1658 } 1659 spin_unlock_bh(&t->stid_lock); 1660 return stid; 1661} 1662EXPORT_SYMBOL(cxgb4_alloc_stid); 1663 1664/* Allocate a server filter TID and set it to the supplied value. 1665 */ 1666int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 1667{ 1668 int stid; 1669 1670 spin_lock_bh(&t->stid_lock); 1671 if (family == PF_INET) { 1672 stid = find_next_zero_bit(t->stid_bmap, 1673 t->nstids + t->nsftids, t->nstids); 1674 if (stid < (t->nstids + t->nsftids)) 1675 __set_bit(stid, t->stid_bmap); 1676 else 1677 stid = -1; 1678 } else { 1679 stid = -1; 1680 } 1681 if (stid >= 0) { 1682 t->stid_tab[stid].data = data; 1683 stid -= t->nstids; 1684 stid += t->sftid_base; 1685 t->sftids_in_use++; 1686 } 1687 spin_unlock_bh(&t->stid_lock); 1688 return stid; 1689} 1690EXPORT_SYMBOL(cxgb4_alloc_sftid); 1691 1692/* Release a server TID. 1693 */ 1694void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 1695{ 1696 /* Is it a server filter TID? */ 1697 if (t->nsftids && (stid >= t->sftid_base)) { 1698 stid -= t->sftid_base; 1699 stid += t->nstids; 1700 } else { 1701 stid -= t->stid_base; 1702 } 1703 1704 spin_lock_bh(&t->stid_lock); 1705 if (family == PF_INET) 1706 __clear_bit(stid, t->stid_bmap); 1707 else 1708 bitmap_release_region(t->stid_bmap, stid, 1); 1709 t->stid_tab[stid].data = NULL; 1710 if (stid < t->nstids) { 1711 if (family == PF_INET6) { 1712 t->stids_in_use -= 2; 1713 t->v6_stids_in_use -= 2; 1714 } else { 1715 t->stids_in_use--; 1716 } 1717 } else { 1718 t->sftids_in_use--; 1719 } 1720 1721 spin_unlock_bh(&t->stid_lock); 1722} 1723EXPORT_SYMBOL(cxgb4_free_stid); 1724 1725/* 1726 * Populate a TID_RELEASE WR. Caller must properly size the skb. 1727 */ 1728static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 1729 unsigned int tid) 1730{ 1731 struct cpl_tid_release *req; 1732 1733 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1734 req = __skb_put(skb, sizeof(*req)); 1735 INIT_TP_WR(req, tid); 1736 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 1737} 1738 1739/* 1740 * Queue a TID release request and if necessary schedule a work queue to 1741 * process it. 1742 */ 1743static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 1744 unsigned int tid) 1745{ 1746 struct adapter *adap = container_of(t, struct adapter, tids); 1747 void **p = &t->tid_tab[tid - t->tid_base]; 1748 1749 spin_lock_bh(&adap->tid_release_lock); 1750 *p = adap->tid_release_head; 1751 /* Low 2 bits encode the Tx channel number */ 1752 adap->tid_release_head = (void **)((uintptr_t)p | chan); 1753 if (!adap->tid_release_task_busy) { 1754 adap->tid_release_task_busy = true; 1755 queue_work(adap->workq, &adap->tid_release_task); 1756 } 1757 spin_unlock_bh(&adap->tid_release_lock); 1758} 1759 1760/* 1761 * Process the list of pending TID release requests. 1762 */ 1763static void process_tid_release_list(struct work_struct *work) 1764{ 1765 struct sk_buff *skb; 1766 struct adapter *adap; 1767 1768 adap = container_of(work, struct adapter, tid_release_task); 1769 1770 spin_lock_bh(&adap->tid_release_lock); 1771 while (adap->tid_release_head) { 1772 void **p = adap->tid_release_head; 1773 unsigned int chan = (uintptr_t)p & 3; 1774 p = (void *)p - chan; 1775 1776 adap->tid_release_head = *p; 1777 *p = NULL; 1778 spin_unlock_bh(&adap->tid_release_lock); 1779 1780 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 1781 GFP_KERNEL))) 1782 schedule_timeout_uninterruptible(1); 1783 1784 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 1785 t4_ofld_send(adap, skb); 1786 spin_lock_bh(&adap->tid_release_lock); 1787 } 1788 adap->tid_release_task_busy = false; 1789 spin_unlock_bh(&adap->tid_release_lock); 1790} 1791 1792/* 1793 * Release a TID and inform HW. If we are unable to allocate the release 1794 * message we defer to a work queue. 1795 */ 1796void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, 1797 unsigned short family) 1798{ 1799 struct adapter *adap = container_of(t, struct adapter, tids); 1800 struct sk_buff *skb; 1801 1802 if (tid_out_of_range(&adap->tids, tid)) { 1803 dev_err(adap->pdev_dev, "tid %d out of range\n", tid); 1804 return; 1805 } 1806 1807 if (t->tid_tab[tid - adap->tids.tid_base]) { 1808 t->tid_tab[tid - adap->tids.tid_base] = NULL; 1809 atomic_dec(&t->conns_in_use); 1810 if (t->hash_base && (tid >= t->hash_base)) { 1811 if (family == AF_INET6) 1812 atomic_sub(2, &t->hash_tids_in_use); 1813 else 1814 atomic_dec(&t->hash_tids_in_use); 1815 } else { 1816 if (family == AF_INET6) 1817 atomic_sub(2, &t->tids_in_use); 1818 else 1819 atomic_dec(&t->tids_in_use); 1820 } 1821 } 1822 1823 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 1824 if (likely(skb)) { 1825 mk_tid_release(skb, chan, tid); 1826 t4_ofld_send(adap, skb); 1827 } else 1828 cxgb4_queue_tid_release(t, chan, tid); 1829} 1830EXPORT_SYMBOL(cxgb4_remove_tid); 1831 1832/* 1833 * Allocate and initialize the TID tables. Returns 0 on success. 1834 */ 1835static int tid_init(struct tid_info *t) 1836{ 1837 struct adapter *adap = container_of(t, struct adapter, tids); 1838 unsigned int max_ftids = t->nftids + t->nsftids; 1839 unsigned int natids = t->natids; 1840 unsigned int hpftid_bmap_size; 1841 unsigned int eotid_bmap_size; 1842 unsigned int stid_bmap_size; 1843 unsigned int ftid_bmap_size; 1844 size_t size; 1845 1846 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 1847 ftid_bmap_size = BITS_TO_LONGS(t->nftids); 1848 hpftid_bmap_size = BITS_TO_LONGS(t->nhpftids); 1849 eotid_bmap_size = BITS_TO_LONGS(t->neotids); 1850 size = t->ntids * sizeof(*t->tid_tab) + 1851 natids * sizeof(*t->atid_tab) + 1852 t->nstids * sizeof(*t->stid_tab) + 1853 t->nsftids * sizeof(*t->stid_tab) + 1854 stid_bmap_size * sizeof(long) + 1855 t->nhpftids * sizeof(*t->hpftid_tab) + 1856 hpftid_bmap_size * sizeof(long) + 1857 max_ftids * sizeof(*t->ftid_tab) + 1858 ftid_bmap_size * sizeof(long) + 1859 t->neotids * sizeof(*t->eotid_tab) + 1860 eotid_bmap_size * sizeof(long); 1861 1862 t->tid_tab = kvzalloc(size, GFP_KERNEL); 1863 if (!t->tid_tab) 1864 return -ENOMEM; 1865 1866 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 1867 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 1868 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 1869 t->hpftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 1870 t->hpftid_bmap = (unsigned long *)&t->hpftid_tab[t->nhpftids]; 1871 t->ftid_tab = (struct filter_entry *)&t->hpftid_bmap[hpftid_bmap_size]; 1872 t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids]; 1873 t->eotid_tab = (struct eotid_entry *)&t->ftid_bmap[ftid_bmap_size]; 1874 t->eotid_bmap = (unsigned long *)&t->eotid_tab[t->neotids]; 1875 spin_lock_init(&t->stid_lock); 1876 spin_lock_init(&t->atid_lock); 1877 spin_lock_init(&t->ftid_lock); 1878 1879 t->stids_in_use = 0; 1880 t->v6_stids_in_use = 0; 1881 t->sftids_in_use = 0; 1882 t->afree = NULL; 1883 t->atids_in_use = 0; 1884 atomic_set(&t->tids_in_use, 0); 1885 atomic_set(&t->conns_in_use, 0); 1886 atomic_set(&t->hash_tids_in_use, 0); 1887 atomic_set(&t->eotids_in_use, 0); 1888 1889 /* Setup the free list for atid_tab and clear the stid bitmap. */ 1890 if (natids) { 1891 while (--natids) 1892 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1893 t->afree = t->atid_tab; 1894 } 1895 1896 if (is_offload(adap)) { 1897 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 1898 /* Reserve stid 0 for T4/T5 adapters */ 1899 if (!t->stid_base && 1900 CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1901 __set_bit(0, t->stid_bmap); 1902 1903 if (t->neotids) 1904 bitmap_zero(t->eotid_bmap, t->neotids); 1905 } 1906 1907 if (t->nhpftids) 1908 bitmap_zero(t->hpftid_bmap, t->nhpftids); 1909 bitmap_zero(t->ftid_bmap, t->nftids); 1910 return 0; 1911} 1912 1913/** 1914 * cxgb4_create_server - create an IP server 1915 * @dev: the device 1916 * @stid: the server TID 1917 * @sip: local IP address to bind server to 1918 * @sport: the server's TCP port 1919 * @vlan: the VLAN header information 1920 * @queue: queue to direct messages from this server to 1921 * 1922 * Create an IP server for the given port and address. 1923 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1924 */ 1925int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 1926 __be32 sip, __be16 sport, __be16 vlan, 1927 unsigned int queue) 1928{ 1929 unsigned int chan; 1930 struct sk_buff *skb; 1931 struct adapter *adap; 1932 struct cpl_pass_open_req *req; 1933 int ret; 1934 1935 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1936 if (!skb) 1937 return -ENOMEM; 1938 1939 adap = netdev2adap(dev); 1940 req = __skb_put(skb, sizeof(*req)); 1941 INIT_TP_WR(req, 0); 1942 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 1943 req->local_port = sport; 1944 req->peer_port = htons(0); 1945 req->local_ip = sip; 1946 req->peer_ip = htonl(0); 1947 chan = rxq_to_chan(&adap->sge, queue); 1948 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1949 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1950 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1951 ret = t4_mgmt_tx(adap, skb); 1952 return net_xmit_eval(ret); 1953} 1954EXPORT_SYMBOL(cxgb4_create_server); 1955 1956/* cxgb4_create_server6 - create an IPv6 server 1957 * @dev: the device 1958 * @stid: the server TID 1959 * @sip: local IPv6 address to bind server to 1960 * @sport: the server's TCP port 1961 * @queue: queue to direct messages from this server to 1962 * 1963 * Create an IPv6 server for the given port and address. 1964 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1965 */ 1966int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 1967 const struct in6_addr *sip, __be16 sport, 1968 unsigned int queue) 1969{ 1970 unsigned int chan; 1971 struct sk_buff *skb; 1972 struct adapter *adap; 1973 struct cpl_pass_open_req6 *req; 1974 int ret; 1975 1976 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1977 if (!skb) 1978 return -ENOMEM; 1979 1980 adap = netdev2adap(dev); 1981 req = __skb_put(skb, sizeof(*req)); 1982 INIT_TP_WR(req, 0); 1983 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 1984 req->local_port = sport; 1985 req->peer_port = htons(0); 1986 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 1987 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 1988 req->peer_ip_hi = cpu_to_be64(0); 1989 req->peer_ip_lo = cpu_to_be64(0); 1990 chan = rxq_to_chan(&adap->sge, queue); 1991 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1992 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1993 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1994 ret = t4_mgmt_tx(adap, skb); 1995 return net_xmit_eval(ret); 1996} 1997EXPORT_SYMBOL(cxgb4_create_server6); 1998 1999int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 2000 unsigned int queue, bool ipv6) 2001{ 2002 struct sk_buff *skb; 2003 struct adapter *adap; 2004 struct cpl_close_listsvr_req *req; 2005 int ret; 2006 2007 adap = netdev2adap(dev); 2008 2009 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 2010 if (!skb) 2011 return -ENOMEM; 2012 2013 req = __skb_put(skb, sizeof(*req)); 2014 INIT_TP_WR(req, 0); 2015 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 2016 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 2017 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 2018 ret = t4_mgmt_tx(adap, skb); 2019 return net_xmit_eval(ret); 2020} 2021EXPORT_SYMBOL(cxgb4_remove_server); 2022 2023/** 2024 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 2025 * @mtus: the HW MTU table 2026 * @mtu: the target MTU 2027 * @idx: index of selected entry in the MTU table 2028 * 2029 * Returns the index and the value in the HW MTU table that is closest to 2030 * but does not exceed @mtu, unless @mtu is smaller than any value in the 2031 * table, in which case that smallest available value is selected. 2032 */ 2033unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 2034 unsigned int *idx) 2035{ 2036 unsigned int i = 0; 2037 2038 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 2039 ++i; 2040 if (idx) 2041 *idx = i; 2042 return mtus[i]; 2043} 2044EXPORT_SYMBOL(cxgb4_best_mtu); 2045 2046/** 2047 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 2048 * @mtus: the HW MTU table 2049 * @header_size: Header Size 2050 * @data_size_max: maximum Data Segment Size 2051 * @data_size_align: desired Data Segment Size Alignment (2^N) 2052 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 2053 * 2054 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 2055 * MTU Table based solely on a Maximum MTU parameter, we break that 2056 * parameter up into a Header Size and Maximum Data Segment Size, and 2057 * provide a desired Data Segment Size Alignment. If we find an MTU in 2058 * the Hardware MTU Table which will result in a Data Segment Size with 2059 * the requested alignment _and_ that MTU isn't "too far" from the 2060 * closest MTU, then we'll return that rather than the closest MTU. 2061 */ 2062unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 2063 unsigned short header_size, 2064 unsigned short data_size_max, 2065 unsigned short data_size_align, 2066 unsigned int *mtu_idxp) 2067{ 2068 unsigned short max_mtu = header_size + data_size_max; 2069 unsigned short data_size_align_mask = data_size_align - 1; 2070 int mtu_idx, aligned_mtu_idx; 2071 2072 /* Scan the MTU Table till we find an MTU which is larger than our 2073 * Maximum MTU or we reach the end of the table. Along the way, 2074 * record the last MTU found, if any, which will result in a Data 2075 * Segment Length matching the requested alignment. 2076 */ 2077 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 2078 unsigned short data_size = mtus[mtu_idx] - header_size; 2079 2080 /* If this MTU minus the Header Size would result in a 2081 * Data Segment Size of the desired alignment, remember it. 2082 */ 2083 if ((data_size & data_size_align_mask) == 0) 2084 aligned_mtu_idx = mtu_idx; 2085 2086 /* If we're not at the end of the Hardware MTU Table and the 2087 * next element is larger than our Maximum MTU, drop out of 2088 * the loop. 2089 */ 2090 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 2091 break; 2092 } 2093 2094 /* If we fell out of the loop because we ran to the end of the table, 2095 * then we just have to use the last [largest] entry. 2096 */ 2097 if (mtu_idx == NMTUS) 2098 mtu_idx--; 2099 2100 /* If we found an MTU which resulted in the requested Data Segment 2101 * Length alignment and that's "not far" from the largest MTU which is 2102 * less than or equal to the maximum MTU, then use that. 2103 */ 2104 if (aligned_mtu_idx >= 0 && 2105 mtu_idx - aligned_mtu_idx <= 1) 2106 mtu_idx = aligned_mtu_idx; 2107 2108 /* If the caller has passed in an MTU Index pointer, pass the 2109 * MTU Index back. Return the MTU value. 2110 */ 2111 if (mtu_idxp) 2112 *mtu_idxp = mtu_idx; 2113 return mtus[mtu_idx]; 2114} 2115EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 2116 2117/** 2118 * cxgb4_port_chan - get the HW channel of a port 2119 * @dev: the net device for the port 2120 * 2121 * Return the HW Tx channel of the given port. 2122 */ 2123unsigned int cxgb4_port_chan(const struct net_device *dev) 2124{ 2125 return netdev2pinfo(dev)->tx_chan; 2126} 2127EXPORT_SYMBOL(cxgb4_port_chan); 2128 2129/** 2130 * cxgb4_port_e2cchan - get the HW c-channel of a port 2131 * @dev: the net device for the port 2132 * 2133 * Return the HW RX c-channel of the given port. 2134 */ 2135unsigned int cxgb4_port_e2cchan(const struct net_device *dev) 2136{ 2137 return netdev2pinfo(dev)->rx_cchan; 2138} 2139EXPORT_SYMBOL(cxgb4_port_e2cchan); 2140 2141unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 2142{ 2143 struct adapter *adap = netdev2adap(dev); 2144 u32 v1, v2, lp_count, hp_count; 2145 2146 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 2147 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 2148 if (is_t4(adap->params.chip)) { 2149 lp_count = LP_COUNT_G(v1); 2150 hp_count = HP_COUNT_G(v1); 2151 } else { 2152 lp_count = LP_COUNT_T5_G(v1); 2153 hp_count = HP_COUNT_T5_G(v2); 2154 } 2155 return lpfifo ? lp_count : hp_count; 2156} 2157EXPORT_SYMBOL(cxgb4_dbfifo_count); 2158 2159/** 2160 * cxgb4_port_viid - get the VI id of a port 2161 * @dev: the net device for the port 2162 * 2163 * Return the VI id of the given port. 2164 */ 2165unsigned int cxgb4_port_viid(const struct net_device *dev) 2166{ 2167 return netdev2pinfo(dev)->viid; 2168} 2169EXPORT_SYMBOL(cxgb4_port_viid); 2170 2171/** 2172 * cxgb4_port_idx - get the index of a port 2173 * @dev: the net device for the port 2174 * 2175 * Return the index of the given port. 2176 */ 2177unsigned int cxgb4_port_idx(const struct net_device *dev) 2178{ 2179 return netdev2pinfo(dev)->port_id; 2180} 2181EXPORT_SYMBOL(cxgb4_port_idx); 2182 2183void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 2184 struct tp_tcp_stats *v6) 2185{ 2186 struct adapter *adap = pci_get_drvdata(pdev); 2187 2188 spin_lock(&adap->stats_lock); 2189 t4_tp_get_tcp_stats(adap, v4, v6, false); 2190 spin_unlock(&adap->stats_lock); 2191} 2192EXPORT_SYMBOL(cxgb4_get_tcp_stats); 2193 2194int cxgb4_flush_eq_cache(struct net_device *dev) 2195{ 2196 struct adapter *adap = netdev2adap(dev); 2197 2198 return t4_sge_ctxt_flush(adap, adap->mbox, CTXT_EGRESS); 2199} 2200EXPORT_SYMBOL(cxgb4_flush_eq_cache); 2201 2202static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 2203{ 2204 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 2205 __be64 indices; 2206 int ret; 2207 2208 spin_lock(&adap->win0_lock); 2209 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 2210 sizeof(indices), (__be32 *)&indices, 2211 T4_MEMORY_READ); 2212 spin_unlock(&adap->win0_lock); 2213 if (!ret) { 2214 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 2215 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 2216 } 2217 return ret; 2218} 2219 2220int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 2221 u16 size) 2222{ 2223 struct adapter *adap = netdev2adap(dev); 2224 u16 hw_pidx, hw_cidx; 2225 int ret; 2226 2227 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 2228 if (ret) 2229 goto out; 2230 2231 if (pidx != hw_pidx) { 2232 u16 delta; 2233 u32 val; 2234 2235 if (pidx >= hw_pidx) 2236 delta = pidx - hw_pidx; 2237 else 2238 delta = size - hw_pidx + pidx; 2239 2240 if (is_t4(adap->params.chip)) 2241 val = PIDX_V(delta); 2242 else 2243 val = PIDX_T5_V(delta); 2244 wmb(); 2245 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2246 QID_V(qid) | val); 2247 } 2248out: 2249 return ret; 2250} 2251EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 2252 2253int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 2254{ 2255 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 2256 u32 edc0_end, edc1_end, mc0_end, mc1_end; 2257 u32 offset, memtype, memaddr; 2258 struct adapter *adap; 2259 u32 hma_size = 0; 2260 int ret; 2261 2262 adap = netdev2adap(dev); 2263 2264 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 2265 2266 /* Figure out where the offset lands in the Memory Type/Address scheme. 2267 * This code assumes that the memory is laid out starting at offset 0 2268 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 2269 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 2270 * MC0, and some have both MC0 and MC1. 2271 */ 2272 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 2273 edc0_size = EDRAM0_SIZE_G(size) << 20; 2274 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 2275 edc1_size = EDRAM1_SIZE_G(size) << 20; 2276 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 2277 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 2278 2279 if (t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A) & HMA_MUX_F) { 2280 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 2281 hma_size = EXT_MEM1_SIZE_G(size) << 20; 2282 } 2283 edc0_end = edc0_size; 2284 edc1_end = edc0_end + edc1_size; 2285 mc0_end = edc1_end + mc0_size; 2286 2287 if (offset < edc0_end) { 2288 memtype = MEM_EDC0; 2289 memaddr = offset; 2290 } else if (offset < edc1_end) { 2291 memtype = MEM_EDC1; 2292 memaddr = offset - edc0_end; 2293 } else { 2294 if (hma_size && (offset < (edc1_end + hma_size))) { 2295 memtype = MEM_HMA; 2296 memaddr = offset - edc1_end; 2297 } else if (offset < mc0_end) { 2298 memtype = MEM_MC0; 2299 memaddr = offset - edc1_end; 2300 } else if (is_t5(adap->params.chip)) { 2301 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 2302 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 2303 mc1_end = mc0_end + mc1_size; 2304 if (offset < mc1_end) { 2305 memtype = MEM_MC1; 2306 memaddr = offset - mc0_end; 2307 } else { 2308 /* offset beyond the end of any memory */ 2309 goto err; 2310 } 2311 } else { 2312 /* T4/T6 only has a single memory channel */ 2313 goto err; 2314 } 2315 } 2316 2317 spin_lock(&adap->win0_lock); 2318 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 2319 spin_unlock(&adap->win0_lock); 2320 return ret; 2321 2322err: 2323 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 2324 stag, offset); 2325 return -EINVAL; 2326} 2327EXPORT_SYMBOL(cxgb4_read_tpte); 2328 2329u64 cxgb4_read_sge_timestamp(struct net_device *dev) 2330{ 2331 u32 hi, lo; 2332 struct adapter *adap; 2333 2334 adap = netdev2adap(dev); 2335 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 2336 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 2337 2338 return ((u64)hi << 32) | (u64)lo; 2339} 2340EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 2341 2342int cxgb4_bar2_sge_qregs(struct net_device *dev, 2343 unsigned int qid, 2344 enum cxgb4_bar2_qtype qtype, 2345 int user, 2346 u64 *pbar2_qoffset, 2347 unsigned int *pbar2_qid) 2348{ 2349 return t4_bar2_sge_qregs(netdev2adap(dev), 2350 qid, 2351 (qtype == CXGB4_BAR2_QTYPE_EGRESS 2352 ? T4_BAR2_QTYPE_EGRESS 2353 : T4_BAR2_QTYPE_INGRESS), 2354 user, 2355 pbar2_qoffset, 2356 pbar2_qid); 2357} 2358EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 2359 2360static struct pci_driver cxgb4_driver; 2361 2362static void check_neigh_update(struct neighbour *neigh) 2363{ 2364 const struct device *parent; 2365 const struct net_device *netdev = neigh->dev; 2366 2367 if (is_vlan_dev(netdev)) 2368 netdev = vlan_dev_real_dev(netdev); 2369 parent = netdev->dev.parent; 2370 if (parent && parent->driver == &cxgb4_driver.driver) 2371 t4_l2t_update(dev_get_drvdata(parent), neigh); 2372} 2373 2374static int netevent_cb(struct notifier_block *nb, unsigned long event, 2375 void *data) 2376{ 2377 switch (event) { 2378 case NETEVENT_NEIGH_UPDATE: 2379 check_neigh_update(data); 2380 break; 2381 case NETEVENT_REDIRECT: 2382 default: 2383 break; 2384 } 2385 return 0; 2386} 2387 2388static bool netevent_registered; 2389static struct notifier_block cxgb4_netevent_nb = { 2390 .notifier_call = netevent_cb 2391}; 2392 2393static void drain_db_fifo(struct adapter *adap, int usecs) 2394{ 2395 u32 v1, v2, lp_count, hp_count; 2396 2397 do { 2398 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 2399 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 2400 if (is_t4(adap->params.chip)) { 2401 lp_count = LP_COUNT_G(v1); 2402 hp_count = HP_COUNT_G(v1); 2403 } else { 2404 lp_count = LP_COUNT_T5_G(v1); 2405 hp_count = HP_COUNT_T5_G(v2); 2406 } 2407 2408 if (lp_count == 0 && hp_count == 0) 2409 break; 2410 set_current_state(TASK_UNINTERRUPTIBLE); 2411 schedule_timeout(usecs_to_jiffies(usecs)); 2412 } while (1); 2413} 2414 2415static void disable_txq_db(struct sge_txq *q) 2416{ 2417 unsigned long flags; 2418 2419 spin_lock_irqsave(&q->db_lock, flags); 2420 q->db_disabled = 1; 2421 spin_unlock_irqrestore(&q->db_lock, flags); 2422} 2423 2424static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 2425{ 2426 spin_lock_irq(&q->db_lock); 2427 if (q->db_pidx_inc) { 2428 /* Make sure that all writes to the TX descriptors 2429 * are committed before we tell HW about them. 2430 */ 2431 wmb(); 2432 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2433 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 2434 q->db_pidx_inc = 0; 2435 } 2436 q->db_disabled = 0; 2437 spin_unlock_irq(&q->db_lock); 2438} 2439 2440static void disable_dbs(struct adapter *adap) 2441{ 2442 int i; 2443 2444 for_each_ethrxq(&adap->sge, i) 2445 disable_txq_db(&adap->sge.ethtxq[i].q); 2446 if (is_offload(adap)) { 2447 struct sge_uld_txq_info *txq_info = 2448 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2449 2450 if (txq_info) { 2451 for_each_ofldtxq(&adap->sge, i) { 2452 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2453 2454 disable_txq_db(&txq->q); 2455 } 2456 } 2457 } 2458 for_each_port(adap, i) 2459 disable_txq_db(&adap->sge.ctrlq[i].q); 2460} 2461 2462static void enable_dbs(struct adapter *adap) 2463{ 2464 int i; 2465 2466 for_each_ethrxq(&adap->sge, i) 2467 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 2468 if (is_offload(adap)) { 2469 struct sge_uld_txq_info *txq_info = 2470 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2471 2472 if (txq_info) { 2473 for_each_ofldtxq(&adap->sge, i) { 2474 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2475 2476 enable_txq_db(adap, &txq->q); 2477 } 2478 } 2479 } 2480 for_each_port(adap, i) 2481 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 2482} 2483 2484static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 2485{ 2486 enum cxgb4_uld type = CXGB4_ULD_RDMA; 2487 2488 if (adap->uld && adap->uld[type].handle) 2489 adap->uld[type].control(adap->uld[type].handle, cmd); 2490} 2491 2492static void process_db_full(struct work_struct *work) 2493{ 2494 struct adapter *adap; 2495 2496 adap = container_of(work, struct adapter, db_full_task); 2497 2498 drain_db_fifo(adap, dbfifo_drain_delay); 2499 enable_dbs(adap); 2500 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2501 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2502 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2503 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 2504 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 2505 else 2506 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2507 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); 2508} 2509 2510static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 2511{ 2512 u16 hw_pidx, hw_cidx; 2513 int ret; 2514 2515 spin_lock_irq(&q->db_lock); 2516 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 2517 if (ret) 2518 goto out; 2519 if (q->db_pidx != hw_pidx) { 2520 u16 delta; 2521 u32 val; 2522 2523 if (q->db_pidx >= hw_pidx) 2524 delta = q->db_pidx - hw_pidx; 2525 else 2526 delta = q->size - hw_pidx + q->db_pidx; 2527 2528 if (is_t4(adap->params.chip)) 2529 val = PIDX_V(delta); 2530 else 2531 val = PIDX_T5_V(delta); 2532 wmb(); 2533 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2534 QID_V(q->cntxt_id) | val); 2535 } 2536out: 2537 q->db_disabled = 0; 2538 q->db_pidx_inc = 0; 2539 spin_unlock_irq(&q->db_lock); 2540 if (ret) 2541 CH_WARN(adap, "DB drop recovery failed.\n"); 2542} 2543 2544static void recover_all_queues(struct adapter *adap) 2545{ 2546 int i; 2547 2548 for_each_ethrxq(&adap->sge, i) 2549 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 2550 if (is_offload(adap)) { 2551 struct sge_uld_txq_info *txq_info = 2552 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2553 if (txq_info) { 2554 for_each_ofldtxq(&adap->sge, i) { 2555 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2556 2557 sync_txq_pidx(adap, &txq->q); 2558 } 2559 } 2560 } 2561 for_each_port(adap, i) 2562 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 2563} 2564 2565static void process_db_drop(struct work_struct *work) 2566{ 2567 struct adapter *adap; 2568 2569 adap = container_of(work, struct adapter, db_drop_task); 2570 2571 if (is_t4(adap->params.chip)) { 2572 drain_db_fifo(adap, dbfifo_drain_delay); 2573 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 2574 drain_db_fifo(adap, dbfifo_drain_delay); 2575 recover_all_queues(adap); 2576 drain_db_fifo(adap, dbfifo_drain_delay); 2577 enable_dbs(adap); 2578 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2579 } else if (is_t5(adap->params.chip)) { 2580 u32 dropped_db = t4_read_reg(adap, 0x010ac); 2581 u16 qid = (dropped_db >> 15) & 0x1ffff; 2582 u16 pidx_inc = dropped_db & 0x1fff; 2583 u64 bar2_qoffset; 2584 unsigned int bar2_qid; 2585 int ret; 2586 2587 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 2588 0, &bar2_qoffset, &bar2_qid); 2589 if (ret) 2590 dev_err(adap->pdev_dev, "doorbell drop recovery: " 2591 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 2592 else 2593 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 2594 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 2595 2596 /* Re-enable BAR2 WC */ 2597 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 2598 } 2599 2600 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2601 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 2602} 2603 2604void t4_db_full(struct adapter *adap) 2605{ 2606 if (is_t4(adap->params.chip)) { 2607 disable_dbs(adap); 2608 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2609 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2610 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 2611 queue_work(adap->workq, &adap->db_full_task); 2612 } 2613} 2614 2615void t4_db_dropped(struct adapter *adap) 2616{ 2617 if (is_t4(adap->params.chip)) { 2618 disable_dbs(adap); 2619 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2620 } 2621 queue_work(adap->workq, &adap->db_drop_task); 2622} 2623 2624void t4_register_netevent_notifier(void) 2625{ 2626 if (!netevent_registered) { 2627 register_netevent_notifier(&cxgb4_netevent_nb); 2628 netevent_registered = true; 2629 } 2630} 2631 2632static void detach_ulds(struct adapter *adap) 2633{ 2634 unsigned int i; 2635 2636 if (!is_uld(adap)) 2637 return; 2638 2639 mutex_lock(&uld_mutex); 2640 list_del(&adap->list_node); 2641 2642 for (i = 0; i < CXGB4_ULD_MAX; i++) 2643 if (adap->uld && adap->uld[i].handle) 2644 adap->uld[i].state_change(adap->uld[i].handle, 2645 CXGB4_STATE_DETACH); 2646 2647 if (netevent_registered && list_empty(&adapter_list)) { 2648 unregister_netevent_notifier(&cxgb4_netevent_nb); 2649 netevent_registered = false; 2650 } 2651 mutex_unlock(&uld_mutex); 2652} 2653 2654static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 2655{ 2656 unsigned int i; 2657 2658 mutex_lock(&uld_mutex); 2659 for (i = 0; i < CXGB4_ULD_MAX; i++) 2660 if (adap->uld && adap->uld[i].handle) 2661 adap->uld[i].state_change(adap->uld[i].handle, 2662 new_state); 2663 mutex_unlock(&uld_mutex); 2664} 2665 2666#if IS_ENABLED(CONFIG_IPV6) 2667static int cxgb4_inet6addr_handler(struct notifier_block *this, 2668 unsigned long event, void *data) 2669{ 2670 struct inet6_ifaddr *ifa = data; 2671 struct net_device *event_dev = ifa->idev->dev; 2672 const struct device *parent = NULL; 2673#if IS_ENABLED(CONFIG_BONDING) 2674 struct adapter *adap; 2675#endif 2676 if (is_vlan_dev(event_dev)) 2677 event_dev = vlan_dev_real_dev(event_dev); 2678#if IS_ENABLED(CONFIG_BONDING) 2679 if (event_dev->flags & IFF_MASTER) { 2680 list_for_each_entry(adap, &adapter_list, list_node) { 2681 switch (event) { 2682 case NETDEV_UP: 2683 cxgb4_clip_get(adap->port[0], 2684 (const u32 *)ifa, 1); 2685 break; 2686 case NETDEV_DOWN: 2687 cxgb4_clip_release(adap->port[0], 2688 (const u32 *)ifa, 1); 2689 break; 2690 default: 2691 break; 2692 } 2693 } 2694 return NOTIFY_OK; 2695 } 2696#endif 2697 2698 if (event_dev) 2699 parent = event_dev->dev.parent; 2700 2701 if (parent && parent->driver == &cxgb4_driver.driver) { 2702 switch (event) { 2703 case NETDEV_UP: 2704 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 2705 break; 2706 case NETDEV_DOWN: 2707 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 2708 break; 2709 default: 2710 break; 2711 } 2712 } 2713 return NOTIFY_OK; 2714} 2715 2716static bool inet6addr_registered; 2717static struct notifier_block cxgb4_inet6addr_notifier = { 2718 .notifier_call = cxgb4_inet6addr_handler 2719}; 2720 2721static void update_clip(const struct adapter *adap) 2722{ 2723 int i; 2724 struct net_device *dev; 2725 int ret; 2726 2727 rcu_read_lock(); 2728 2729 for (i = 0; i < MAX_NPORTS; i++) { 2730 dev = adap->port[i]; 2731 ret = 0; 2732 2733 if (dev) 2734 ret = cxgb4_update_root_dev_clip(dev); 2735 2736 if (ret < 0) 2737 break; 2738 } 2739 rcu_read_unlock(); 2740} 2741#endif /* IS_ENABLED(CONFIG_IPV6) */ 2742 2743/** 2744 * cxgb_up - enable the adapter 2745 * @adap: adapter being enabled 2746 * 2747 * Called when the first port is enabled, this function performs the 2748 * actions necessary to make an adapter operational, such as completing 2749 * the initialization of HW modules, and enabling interrupts. 2750 * 2751 * Must be called with the rtnl lock held. 2752 */ 2753static int cxgb_up(struct adapter *adap) 2754{ 2755 struct sge *s = &adap->sge; 2756 int err; 2757 2758 mutex_lock(&uld_mutex); 2759 err = setup_sge_queues(adap); 2760 if (err) 2761 goto rel_lock; 2762 err = setup_rss(adap); 2763 if (err) 2764 goto freeq; 2765 2766 if (adap->flags & CXGB4_USING_MSIX) { 2767 if (s->nd_msix_idx < 0) { 2768 err = -ENOMEM; 2769 goto irq_err; 2770 } 2771 2772 err = request_irq(adap->msix_info[s->nd_msix_idx].vec, 2773 t4_nondata_intr, 0, 2774 adap->msix_info[s->nd_msix_idx].desc, adap); 2775 if (err) 2776 goto irq_err; 2777 2778 err = request_msix_queue_irqs(adap); 2779 if (err) 2780 goto irq_err_free_nd_msix; 2781 } else { 2782 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 2783 (adap->flags & CXGB4_USING_MSI) ? 0 2784 : IRQF_SHARED, 2785 adap->port[0]->name, adap); 2786 if (err) 2787 goto irq_err; 2788 } 2789 2790 enable_rx(adap); 2791 t4_sge_start(adap); 2792 t4_intr_enable(adap); 2793 adap->flags |= CXGB4_FULL_INIT_DONE; 2794 mutex_unlock(&uld_mutex); 2795 2796 notify_ulds(adap, CXGB4_STATE_UP); 2797#if IS_ENABLED(CONFIG_IPV6) 2798 update_clip(adap); 2799#endif 2800 return err; 2801 2802irq_err_free_nd_msix: 2803 free_irq(adap->msix_info[s->nd_msix_idx].vec, adap); 2804irq_err: 2805 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 2806freeq: 2807 t4_free_sge_resources(adap); 2808rel_lock: 2809 mutex_unlock(&uld_mutex); 2810 return err; 2811} 2812 2813static void cxgb_down(struct adapter *adapter) 2814{ 2815 cancel_work_sync(&adapter->tid_release_task); 2816 cancel_work_sync(&adapter->db_full_task); 2817 cancel_work_sync(&adapter->db_drop_task); 2818 adapter->tid_release_task_busy = false; 2819 adapter->tid_release_head = NULL; 2820 2821 t4_sge_stop(adapter); 2822 t4_free_sge_resources(adapter); 2823 2824 adapter->flags &= ~CXGB4_FULL_INIT_DONE; 2825} 2826 2827/* 2828 * net_device operations 2829 */ 2830static int cxgb_open(struct net_device *dev) 2831{ 2832 struct port_info *pi = netdev_priv(dev); 2833 struct adapter *adapter = pi->adapter; 2834 int err; 2835 2836 netif_carrier_off(dev); 2837 2838 if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) { 2839 err = cxgb_up(adapter); 2840 if (err < 0) 2841 return err; 2842 } 2843 2844 /* It's possible that the basic port information could have 2845 * changed since we first read it. 2846 */ 2847 err = t4_update_port_info(pi); 2848 if (err < 0) 2849 return err; 2850 2851 err = link_start(dev); 2852 if (err) 2853 return err; 2854 2855 if (pi->nmirrorqsets) { 2856 mutex_lock(&pi->vi_mirror_mutex); 2857 err = cxgb4_port_mirror_alloc_queues(dev); 2858 if (err) 2859 goto out_unlock; 2860 2861 err = cxgb4_port_mirror_start(dev); 2862 if (err) 2863 goto out_free_queues; 2864 mutex_unlock(&pi->vi_mirror_mutex); 2865 } 2866 2867 netif_tx_start_all_queues(dev); 2868 return 0; 2869 2870out_free_queues: 2871 cxgb4_port_mirror_free_queues(dev); 2872 2873out_unlock: 2874 mutex_unlock(&pi->vi_mirror_mutex); 2875 return err; 2876} 2877 2878static int cxgb_close(struct net_device *dev) 2879{ 2880 struct port_info *pi = netdev_priv(dev); 2881 struct adapter *adapter = pi->adapter; 2882 int ret; 2883 2884 netif_tx_stop_all_queues(dev); 2885 netif_carrier_off(dev); 2886 ret = t4_enable_pi_params(adapter, adapter->pf, pi, 2887 false, false, false); 2888#ifdef CONFIG_CHELSIO_T4_DCB 2889 cxgb4_dcb_reset(dev); 2890 dcb_tx_queue_prio_enable(dev, false); 2891#endif 2892 if (ret) 2893 return ret; 2894 2895 if (pi->nmirrorqsets) { 2896 mutex_lock(&pi->vi_mirror_mutex); 2897 cxgb4_port_mirror_stop(dev); 2898 cxgb4_port_mirror_free_queues(dev); 2899 mutex_unlock(&pi->vi_mirror_mutex); 2900 } 2901 2902 return 0; 2903} 2904 2905int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 2906 __be32 sip, __be16 sport, __be16 vlan, 2907 unsigned int queue, unsigned char port, unsigned char mask) 2908{ 2909 int ret; 2910 struct filter_entry *f; 2911 struct adapter *adap; 2912 int i; 2913 u8 *val; 2914 2915 adap = netdev2adap(dev); 2916 2917 /* Adjust stid to correct filter index */ 2918 stid -= adap->tids.sftid_base; 2919 stid += adap->tids.nftids; 2920 2921 /* Check to make sure the filter requested is writable ... 2922 */ 2923 f = &adap->tids.ftid_tab[stid]; 2924 ret = writable_filter(f); 2925 if (ret) 2926 return ret; 2927 2928 /* Clear out any old resources being used by the filter before 2929 * we start constructing the new filter. 2930 */ 2931 if (f->valid) 2932 clear_filter(adap, f); 2933 2934 /* Clear out filter specifications */ 2935 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 2936 f->fs.val.lport = be16_to_cpu(sport); 2937 f->fs.mask.lport = ~0; 2938 val = (u8 *)&sip; 2939 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 2940 for (i = 0; i < 4; i++) { 2941 f->fs.val.lip[i] = val[i]; 2942 f->fs.mask.lip[i] = ~0; 2943 } 2944 if (adap->params.tp.vlan_pri_map & PORT_F) { 2945 f->fs.val.iport = port; 2946 f->fs.mask.iport = mask; 2947 } 2948 } 2949 2950 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 2951 f->fs.val.proto = IPPROTO_TCP; 2952 f->fs.mask.proto = ~0; 2953 } 2954 2955 f->fs.dirsteer = 1; 2956 f->fs.iq = queue; 2957 /* Mark filter as locked */ 2958 f->locked = 1; 2959 f->fs.rpttid = 1; 2960 2961 /* Save the actual tid. We need this to get the corresponding 2962 * filter entry structure in filter_rpl. 2963 */ 2964 f->tid = stid + adap->tids.ftid_base; 2965 ret = set_filter_wr(adap, stid); 2966 if (ret) { 2967 clear_filter(adap, f); 2968 return ret; 2969 } 2970 2971 return 0; 2972} 2973EXPORT_SYMBOL(cxgb4_create_server_filter); 2974 2975int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 2976 unsigned int queue, bool ipv6) 2977{ 2978 struct filter_entry *f; 2979 struct adapter *adap; 2980 2981 adap = netdev2adap(dev); 2982 2983 /* Adjust stid to correct filter index */ 2984 stid -= adap->tids.sftid_base; 2985 stid += adap->tids.nftids; 2986 2987 f = &adap->tids.ftid_tab[stid]; 2988 /* Unlock the filter */ 2989 f->locked = 0; 2990 2991 return delete_filter(adap, stid); 2992} 2993EXPORT_SYMBOL(cxgb4_remove_server_filter); 2994 2995static void cxgb_get_stats(struct net_device *dev, 2996 struct rtnl_link_stats64 *ns) 2997{ 2998 struct port_stats stats; 2999 struct port_info *p = netdev_priv(dev); 3000 struct adapter *adapter = p->adapter; 3001 3002 /* Block retrieving statistics during EEH error 3003 * recovery. Otherwise, the recovery might fail 3004 * and the PCI device will be removed permanently 3005 */ 3006 spin_lock(&adapter->stats_lock); 3007 if (!netif_device_present(dev)) { 3008 spin_unlock(&adapter->stats_lock); 3009 return; 3010 } 3011 t4_get_port_stats_offset(adapter, p->tx_chan, &stats, 3012 &p->stats_base); 3013 spin_unlock(&adapter->stats_lock); 3014 3015 ns->tx_bytes = stats.tx_octets; 3016 ns->tx_packets = stats.tx_frames; 3017 ns->rx_bytes = stats.rx_octets; 3018 ns->rx_packets = stats.rx_frames; 3019 ns->multicast = stats.rx_mcast_frames; 3020 3021 /* detailed rx_errors */ 3022 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 3023 stats.rx_runt; 3024 ns->rx_over_errors = 0; 3025 ns->rx_crc_errors = stats.rx_fcs_err; 3026 ns->rx_frame_errors = stats.rx_symbol_err; 3027 ns->rx_dropped = stats.rx_ovflow0 + stats.rx_ovflow1 + 3028 stats.rx_ovflow2 + stats.rx_ovflow3 + 3029 stats.rx_trunc0 + stats.rx_trunc1 + 3030 stats.rx_trunc2 + stats.rx_trunc3; 3031 ns->rx_missed_errors = 0; 3032 3033 /* detailed tx_errors */ 3034 ns->tx_aborted_errors = 0; 3035 ns->tx_carrier_errors = 0; 3036 ns->tx_fifo_errors = 0; 3037 ns->tx_heartbeat_errors = 0; 3038 ns->tx_window_errors = 0; 3039 3040 ns->tx_errors = stats.tx_error_frames; 3041 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 3042 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 3043} 3044 3045static int cxgb_hwtstamp_get(struct net_device *dev, 3046 struct kernel_hwtstamp_config *config) 3047{ 3048 struct port_info *pi = netdev_priv(dev); 3049 3050 *config = pi->tstamp_config; 3051 return 0; 3052} 3053 3054static int cxgb_hwtstamp_set(struct net_device *dev, 3055 struct kernel_hwtstamp_config *config, 3056 struct netlink_ext_ack *extack) 3057{ 3058 struct port_info *pi = netdev_priv(dev); 3059 struct adapter *adapter = pi->adapter; 3060 3061 if (is_t4(adapter->params.chip)) { 3062 /* For T4 Adapters */ 3063 switch (config->rx_filter) { 3064 case HWTSTAMP_FILTER_NONE: 3065 pi->rxtstamp = false; 3066 break; 3067 case HWTSTAMP_FILTER_ALL: 3068 pi->rxtstamp = true; 3069 break; 3070 default: 3071 return -ERANGE; 3072 } 3073 pi->tstamp_config = *config; 3074 return 0; 3075 } 3076 3077 switch (config->tx_type) { 3078 case HWTSTAMP_TX_OFF: 3079 case HWTSTAMP_TX_ON: 3080 break; 3081 default: 3082 return -ERANGE; 3083 } 3084 3085 switch (config->rx_filter) { 3086 case HWTSTAMP_FILTER_NONE: 3087 pi->rxtstamp = false; 3088 break; 3089 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3090 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3091 cxgb4_ptprx_timestamping(pi, pi->port_id, PTP_TS_L4); 3092 break; 3093 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3094 cxgb4_ptprx_timestamping(pi, pi->port_id, PTP_TS_L2_L4); 3095 break; 3096 case HWTSTAMP_FILTER_ALL: 3097 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3098 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3099 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3100 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3101 pi->rxtstamp = true; 3102 break; 3103 default: 3104 return -ERANGE; 3105 } 3106 3107 if (config->tx_type == HWTSTAMP_TX_OFF && 3108 config->rx_filter == HWTSTAMP_FILTER_NONE) { 3109 if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0) 3110 pi->ptp_enable = false; 3111 } 3112 3113 if (config->rx_filter != HWTSTAMP_FILTER_NONE) { 3114 if (cxgb4_ptp_redirect_rx_packet(adapter, pi) >= 0) 3115 pi->ptp_enable = true; 3116 } 3117 pi->tstamp_config = *config; 3118 return 0; 3119} 3120 3121static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 3122{ 3123 unsigned int mbox; 3124 int ret = 0, prtad, devad; 3125 struct port_info *pi = netdev_priv(dev); 3126 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 3127 3128 switch (cmd) { 3129 case SIOCGMIIPHY: 3130 if (pi->mdio_addr < 0) 3131 return -EOPNOTSUPP; 3132 data->phy_id = pi->mdio_addr; 3133 break; 3134 case SIOCGMIIREG: 3135 case SIOCSMIIREG: 3136 if (mdio_phy_id_is_c45(data->phy_id)) { 3137 prtad = mdio_phy_id_prtad(data->phy_id); 3138 devad = mdio_phy_id_devad(data->phy_id); 3139 } else if (data->phy_id < 32) { 3140 prtad = data->phy_id; 3141 devad = 0; 3142 data->reg_num &= 0x1f; 3143 } else 3144 return -EINVAL; 3145 3146 mbox = pi->adapter->pf; 3147 if (cmd == SIOCGMIIREG) 3148 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 3149 data->reg_num, &data->val_out); 3150 else 3151 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 3152 data->reg_num, data->val_in); 3153 break; 3154 default: 3155 return -EOPNOTSUPP; 3156 } 3157 return ret; 3158} 3159 3160static void cxgb_set_rxmode(struct net_device *dev) 3161{ 3162 /* unfortunately we can't return errors to the stack */ 3163 set_rxmode(dev, -1, false); 3164} 3165 3166static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 3167{ 3168 struct port_info *pi = netdev_priv(dev); 3169 int ret; 3170 3171 ret = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid, 3172 pi->viid_mirror, new_mtu, -1, -1, -1, -1, true); 3173 if (!ret) 3174 WRITE_ONCE(dev->mtu, new_mtu); 3175 return ret; 3176} 3177 3178#ifdef CONFIG_PCI_IOV 3179static int cxgb4_mgmt_open(struct net_device *dev) 3180{ 3181 /* Turn carrier off since we don't have to transmit anything on this 3182 * interface. 3183 */ 3184 netif_carrier_off(dev); 3185 return 0; 3186} 3187 3188/* Fill MAC address that will be assigned by the FW */ 3189static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap) 3190{ 3191 u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN]; 3192 unsigned int i, vf, nvfs; 3193 u16 a, b; 3194 int err; 3195 u8 *na; 3196 3197 err = t4_get_raw_vpd_params(adap, &adap->params.vpd); 3198 if (err) 3199 return; 3200 3201 na = adap->params.vpd.na; 3202 for (i = 0; i < ETH_ALEN; i++) 3203 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 3204 hex2val(na[2 * i + 1])); 3205 3206 a = (hw_addr[0] << 8) | hw_addr[1]; 3207 b = (hw_addr[1] << 8) | hw_addr[2]; 3208 a ^= b; 3209 a |= 0x0200; /* locally assigned Ethernet MAC address */ 3210 a &= ~0x0100; /* not a multicast Ethernet MAC address */ 3211 macaddr[0] = a >> 8; 3212 macaddr[1] = a & 0xff; 3213 3214 for (i = 2; i < 5; i++) 3215 macaddr[i] = hw_addr[i + 1]; 3216 3217 for (vf = 0, nvfs = pci_sriov_get_totalvfs(adap->pdev); 3218 vf < nvfs; vf++) { 3219 macaddr[5] = adap->pf * nvfs + vf; 3220 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, macaddr); 3221 } 3222} 3223 3224static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 3225{ 3226 struct port_info *pi = netdev_priv(dev); 3227 struct adapter *adap = pi->adapter; 3228 int ret; 3229 3230 /* verify MAC addr is valid */ 3231 if (!is_valid_ether_addr(mac)) { 3232 dev_err(pi->adapter->pdev_dev, 3233 "Invalid Ethernet address %pM for VF %d\n", 3234 mac, vf); 3235 return -EINVAL; 3236 } 3237 3238 dev_info(pi->adapter->pdev_dev, 3239 "Setting MAC %pM on VF %d\n", mac, vf); 3240 ret = t4_set_vf_mac_acl(adap, vf + 1, pi->lport, 1, mac); 3241 if (!ret) 3242 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac); 3243 return ret; 3244} 3245 3246static int cxgb4_mgmt_get_vf_config(struct net_device *dev, 3247 int vf, struct ifla_vf_info *ivi) 3248{ 3249 struct port_info *pi = netdev_priv(dev); 3250 struct adapter *adap = pi->adapter; 3251 struct vf_info *vfinfo; 3252 3253 if (vf >= adap->num_vfs) 3254 return -EINVAL; 3255 vfinfo = &adap->vfinfo[vf]; 3256 3257 ivi->vf = vf; 3258 ivi->max_tx_rate = vfinfo->tx_rate; 3259 ivi->min_tx_rate = 0; 3260 ether_addr_copy(ivi->mac, vfinfo->vf_mac_addr); 3261 ivi->vlan = vfinfo->vlan; 3262 ivi->linkstate = vfinfo->link_state; 3263 return 0; 3264} 3265 3266static int cxgb4_mgmt_get_phys_port_id(struct net_device *dev, 3267 struct netdev_phys_item_id *ppid) 3268{ 3269 struct port_info *pi = netdev_priv(dev); 3270 unsigned int phy_port_id; 3271 3272 phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id; 3273 ppid->id_len = sizeof(phy_port_id); 3274 memcpy(ppid->id, &phy_port_id, ppid->id_len); 3275 return 0; 3276} 3277 3278static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf, 3279 int min_tx_rate, int max_tx_rate) 3280{ 3281 struct port_info *pi = netdev_priv(dev); 3282 struct adapter *adap = pi->adapter; 3283 unsigned int link_ok, speed, mtu; 3284 u32 fw_pfvf, fw_class; 3285 int class_id = vf; 3286 int ret; 3287 u16 pktsize; 3288 3289 if (vf >= adap->num_vfs) 3290 return -EINVAL; 3291 3292 if (min_tx_rate) { 3293 dev_err(adap->pdev_dev, 3294 "Min tx rate (%d) (> 0) for VF %d is Invalid.\n", 3295 min_tx_rate, vf); 3296 return -EINVAL; 3297 } 3298 3299 if (max_tx_rate == 0) { 3300 /* unbind VF to any Traffic Class */ 3301 fw_pfvf = 3302 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 3303 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); 3304 fw_class = 0xffffffff; 3305 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, 3306 &fw_pfvf, &fw_class); 3307 if (ret) { 3308 dev_err(adap->pdev_dev, 3309 "Err %d in unbinding PF %d VF %d from TX Rate Limiting\n", 3310 ret, adap->pf, vf); 3311 return -EINVAL; 3312 } 3313 dev_info(adap->pdev_dev, 3314 "PF %d VF %d is unbound from TX Rate Limiting\n", 3315 adap->pf, vf); 3316 adap->vfinfo[vf].tx_rate = 0; 3317 return 0; 3318 } 3319 3320 ret = t4_get_link_params(pi, &link_ok, &speed, &mtu); 3321 if (ret != FW_SUCCESS) { 3322 dev_err(adap->pdev_dev, 3323 "Failed to get link information for VF %d\n", vf); 3324 return -EINVAL; 3325 } 3326 3327 if (!link_ok) { 3328 dev_err(adap->pdev_dev, "Link down for VF %d\n", vf); 3329 return -EINVAL; 3330 } 3331 3332 if (max_tx_rate > speed) { 3333 dev_err(adap->pdev_dev, 3334 "Max tx rate %d for VF %d can't be > link-speed %u", 3335 max_tx_rate, vf, speed); 3336 return -EINVAL; 3337 } 3338 3339 pktsize = mtu; 3340 /* subtract ethhdr size and 4 bytes crc since, f/w appends it */ 3341 pktsize = pktsize - sizeof(struct ethhdr) - 4; 3342 /* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */ 3343 pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr); 3344 /* configure Traffic Class for rate-limiting */ 3345 ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET, 3346 SCHED_CLASS_LEVEL_CL_RL, 3347 SCHED_CLASS_MODE_CLASS, 3348 SCHED_CLASS_RATEUNIT_BITS, 3349 SCHED_CLASS_RATEMODE_ABS, 3350 pi->tx_chan, class_id, 0, 3351 max_tx_rate * 1000, 0, pktsize, 0); 3352 if (ret) { 3353 dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n", 3354 ret); 3355 return -EINVAL; 3356 } 3357 dev_info(adap->pdev_dev, 3358 "Class %d with MSS %u configured with rate %u\n", 3359 class_id, pktsize, max_tx_rate); 3360 3361 /* bind VF to configured Traffic Class */ 3362 fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 3363 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); 3364 fw_class = class_id; 3365 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf, 3366 &fw_class); 3367 if (ret) { 3368 dev_err(adap->pdev_dev, 3369 "Err %d in binding PF %d VF %d to Traffic Class %d\n", 3370 ret, adap->pf, vf, class_id); 3371 return -EINVAL; 3372 } 3373 dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n", 3374 adap->pf, vf, class_id); 3375 adap->vfinfo[vf].tx_rate = max_tx_rate; 3376 return 0; 3377} 3378 3379static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf, 3380 u16 vlan, u8 qos, __be16 vlan_proto) 3381{ 3382 struct port_info *pi = netdev_priv(dev); 3383 struct adapter *adap = pi->adapter; 3384 int ret; 3385 3386 if (vf >= adap->num_vfs || vlan > 4095 || qos > 7) 3387 return -EINVAL; 3388 3389 if (vlan_proto != htons(ETH_P_8021Q) || qos != 0) 3390 return -EPROTONOSUPPORT; 3391 3392 ret = t4_set_vlan_acl(adap, adap->mbox, vf + 1, vlan); 3393 if (!ret) { 3394 adap->vfinfo[vf].vlan = vlan; 3395 return 0; 3396 } 3397 3398 dev_err(adap->pdev_dev, "Err %d %s VLAN ACL for PF/VF %d/%d\n", 3399 ret, (vlan ? "setting" : "clearing"), adap->pf, vf); 3400 return ret; 3401} 3402 3403static int cxgb4_mgmt_set_vf_link_state(struct net_device *dev, int vf, 3404 int link) 3405{ 3406 struct port_info *pi = netdev_priv(dev); 3407 struct adapter *adap = pi->adapter; 3408 u32 param, val; 3409 int ret = 0; 3410 3411 if (vf >= adap->num_vfs) 3412 return -EINVAL; 3413 3414 switch (link) { 3415 case IFLA_VF_LINK_STATE_AUTO: 3416 val = FW_VF_LINK_STATE_AUTO; 3417 break; 3418 3419 case IFLA_VF_LINK_STATE_ENABLE: 3420 val = FW_VF_LINK_STATE_ENABLE; 3421 break; 3422 3423 case IFLA_VF_LINK_STATE_DISABLE: 3424 val = FW_VF_LINK_STATE_DISABLE; 3425 break; 3426 3427 default: 3428 return -EINVAL; 3429 } 3430 3431 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 3432 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_LINK_STATE)); 3433 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, 3434 ¶m, &val); 3435 if (ret) { 3436 dev_err(adap->pdev_dev, 3437 "Error %d in setting PF %d VF %d link state\n", 3438 ret, adap->pf, vf); 3439 return -EINVAL; 3440 } 3441 3442 adap->vfinfo[vf].link_state = link; 3443 return ret; 3444} 3445#endif /* CONFIG_PCI_IOV */ 3446 3447static int cxgb_set_mac_addr(struct net_device *dev, void *p) 3448{ 3449 int ret; 3450 struct sockaddr *addr = p; 3451 struct port_info *pi = netdev_priv(dev); 3452 3453 if (!is_valid_ether_addr(addr->sa_data)) 3454 return -EADDRNOTAVAIL; 3455 3456 ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt, 3457 addr->sa_data, true, &pi->smt_idx); 3458 if (ret < 0) 3459 return ret; 3460 3461 eth_hw_addr_set(dev, addr->sa_data); 3462 return 0; 3463} 3464 3465#ifdef CONFIG_NET_POLL_CONTROLLER 3466static void cxgb_netpoll(struct net_device *dev) 3467{ 3468 struct port_info *pi = netdev_priv(dev); 3469 struct adapter *adap = pi->adapter; 3470 3471 if (adap->flags & CXGB4_USING_MSIX) { 3472 int i; 3473 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 3474 3475 for (i = pi->nqsets; i; i--, rx++) 3476 t4_sge_intr_msix(0, &rx->rspq); 3477 } else 3478 t4_intr_handler(adap)(0, adap); 3479} 3480#endif 3481 3482static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 3483{ 3484 struct port_info *pi = netdev_priv(dev); 3485 struct adapter *adap = pi->adapter; 3486 struct ch_sched_queue qe = { 0 }; 3487 struct ch_sched_params p = { 0 }; 3488 struct ch_sched_class *e; 3489 u32 req_rate; 3490 int err = 0; 3491 3492 if (!can_sched(dev)) 3493 return -ENOTSUPP; 3494 3495 if (index < 0 || index > pi->nqsets - 1) 3496 return -EINVAL; 3497 3498 if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { 3499 dev_err(adap->pdev_dev, 3500 "Failed to rate limit on queue %d. Link Down?\n", 3501 index); 3502 return -EINVAL; 3503 } 3504 3505 qe.queue = index; 3506 e = cxgb4_sched_queue_lookup(dev, &qe); 3507 if (e && e->info.u.params.level != SCHED_CLASS_LEVEL_CL_RL) { 3508 dev_err(adap->pdev_dev, 3509 "Queue %u already bound to class %u of type: %u\n", 3510 index, e->idx, e->info.u.params.level); 3511 return -EBUSY; 3512 } 3513 3514 /* Convert from Mbps to Kbps */ 3515 req_rate = rate * 1000; 3516 3517 /* Max rate is 100 Gbps */ 3518 if (req_rate > SCHED_MAX_RATE_KBPS) { 3519 dev_err(adap->pdev_dev, 3520 "Invalid rate %u Mbps, Max rate is %u Mbps\n", 3521 rate, SCHED_MAX_RATE_KBPS / 1000); 3522 return -ERANGE; 3523 } 3524 3525 /* First unbind the queue from any existing class */ 3526 memset(&qe, 0, sizeof(qe)); 3527 qe.queue = index; 3528 qe.class = SCHED_CLS_NONE; 3529 3530 err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE); 3531 if (err) { 3532 dev_err(adap->pdev_dev, 3533 "Unbinding Queue %d on port %d fail. Err: %d\n", 3534 index, pi->port_id, err); 3535 return err; 3536 } 3537 3538 /* Queue already unbound */ 3539 if (!req_rate) 3540 return 0; 3541 3542 /* Fetch any available unused or matching scheduling class */ 3543 p.type = SCHED_CLASS_TYPE_PACKET; 3544 p.u.params.level = SCHED_CLASS_LEVEL_CL_RL; 3545 p.u.params.mode = SCHED_CLASS_MODE_CLASS; 3546 p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS; 3547 p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS; 3548 p.u.params.channel = pi->tx_chan; 3549 p.u.params.class = SCHED_CLS_NONE; 3550 p.u.params.minrate = 0; 3551 p.u.params.maxrate = req_rate; 3552 p.u.params.weight = 0; 3553 p.u.params.pktsize = dev->mtu; 3554 3555 e = cxgb4_sched_class_alloc(dev, &p); 3556 if (!e) 3557 return -ENOMEM; 3558 3559 /* Bind the queue to a scheduling class */ 3560 memset(&qe, 0, sizeof(qe)); 3561 qe.queue = index; 3562 qe.class = e->idx; 3563 3564 err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE); 3565 if (err) 3566 dev_err(adap->pdev_dev, 3567 "Queue rate limiting failed. Err: %d\n", err); 3568 return err; 3569} 3570 3571static int cxgb_setup_tc_flower(struct net_device *dev, 3572 struct flow_cls_offload *cls_flower) 3573{ 3574 switch (cls_flower->command) { 3575 case FLOW_CLS_REPLACE: 3576 return cxgb4_tc_flower_replace(dev, cls_flower); 3577 case FLOW_CLS_DESTROY: 3578 return cxgb4_tc_flower_destroy(dev, cls_flower); 3579 case FLOW_CLS_STATS: 3580 return cxgb4_tc_flower_stats(dev, cls_flower); 3581 default: 3582 return -EOPNOTSUPP; 3583 } 3584} 3585 3586static int cxgb_setup_tc_cls_u32(struct net_device *dev, 3587 struct tc_cls_u32_offload *cls_u32) 3588{ 3589 switch (cls_u32->command) { 3590 case TC_CLSU32_NEW_KNODE: 3591 case TC_CLSU32_REPLACE_KNODE: 3592 return cxgb4_config_knode(dev, cls_u32); 3593 case TC_CLSU32_DELETE_KNODE: 3594 return cxgb4_delete_knode(dev, cls_u32); 3595 default: 3596 return -EOPNOTSUPP; 3597 } 3598} 3599 3600static int cxgb_setup_tc_matchall(struct net_device *dev, 3601 struct tc_cls_matchall_offload *cls_matchall, 3602 bool ingress) 3603{ 3604 struct adapter *adap = netdev2adap(dev); 3605 3606 if (!adap->tc_matchall) 3607 return -ENOMEM; 3608 3609 switch (cls_matchall->command) { 3610 case TC_CLSMATCHALL_REPLACE: 3611 return cxgb4_tc_matchall_replace(dev, cls_matchall, ingress); 3612 case TC_CLSMATCHALL_DESTROY: 3613 return cxgb4_tc_matchall_destroy(dev, cls_matchall, ingress); 3614 case TC_CLSMATCHALL_STATS: 3615 if (ingress) 3616 return cxgb4_tc_matchall_stats(dev, cls_matchall); 3617 break; 3618 default: 3619 break; 3620 } 3621 3622 return -EOPNOTSUPP; 3623} 3624 3625static int cxgb_setup_tc_block_ingress_cb(enum tc_setup_type type, 3626 void *type_data, void *cb_priv) 3627{ 3628 struct net_device *dev = cb_priv; 3629 struct port_info *pi = netdev2pinfo(dev); 3630 struct adapter *adap = netdev2adap(dev); 3631 3632 if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { 3633 dev_err(adap->pdev_dev, 3634 "Failed to setup tc on port %d. Link Down?\n", 3635 pi->port_id); 3636 return -EINVAL; 3637 } 3638 3639 if (!tc_cls_can_offload_and_chain0(dev, type_data)) 3640 return -EOPNOTSUPP; 3641 3642 switch (type) { 3643 case TC_SETUP_CLSU32: 3644 return cxgb_setup_tc_cls_u32(dev, type_data); 3645 case TC_SETUP_CLSFLOWER: 3646 return cxgb_setup_tc_flower(dev, type_data); 3647 case TC_SETUP_CLSMATCHALL: 3648 return cxgb_setup_tc_matchall(dev, type_data, true); 3649 default: 3650 return -EOPNOTSUPP; 3651 } 3652} 3653 3654static int cxgb_setup_tc_block_egress_cb(enum tc_setup_type type, 3655 void *type_data, void *cb_priv) 3656{ 3657 struct net_device *dev = cb_priv; 3658 struct port_info *pi = netdev2pinfo(dev); 3659 struct adapter *adap = netdev2adap(dev); 3660 3661 if (!(adap->flags & CXGB4_FULL_INIT_DONE)) { 3662 dev_err(adap->pdev_dev, 3663 "Failed to setup tc on port %d. Link Down?\n", 3664 pi->port_id); 3665 return -EINVAL; 3666 } 3667 3668 if (!tc_cls_can_offload_and_chain0(dev, type_data)) 3669 return -EOPNOTSUPP; 3670 3671 switch (type) { 3672 case TC_SETUP_CLSMATCHALL: 3673 return cxgb_setup_tc_matchall(dev, type_data, false); 3674 default: 3675 break; 3676 } 3677 3678 return -EOPNOTSUPP; 3679} 3680 3681static int cxgb_setup_tc_mqprio(struct net_device *dev, 3682 struct tc_mqprio_qopt_offload *mqprio) 3683{ 3684 struct adapter *adap = netdev2adap(dev); 3685 3686 if (!is_ethofld(adap) || !adap->tc_mqprio) 3687 return -ENOMEM; 3688 3689 return cxgb4_setup_tc_mqprio(dev, mqprio); 3690} 3691 3692static LIST_HEAD(cxgb_block_cb_list); 3693 3694static int cxgb_setup_tc_block(struct net_device *dev, 3695 struct flow_block_offload *f) 3696{ 3697 struct port_info *pi = netdev_priv(dev); 3698 flow_setup_cb_t *cb; 3699 bool ingress_only; 3700 3701 pi->tc_block_shared = f->block_shared; 3702 if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { 3703 cb = cxgb_setup_tc_block_egress_cb; 3704 ingress_only = false; 3705 } else { 3706 cb = cxgb_setup_tc_block_ingress_cb; 3707 ingress_only = true; 3708 } 3709 3710 return flow_block_cb_setup_simple(f, &cxgb_block_cb_list, 3711 cb, pi, dev, ingress_only); 3712} 3713 3714static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type, 3715 void *type_data) 3716{ 3717 switch (type) { 3718 case TC_SETUP_QDISC_MQPRIO: 3719 return cxgb_setup_tc_mqprio(dev, type_data); 3720 case TC_SETUP_BLOCK: 3721 return cxgb_setup_tc_block(dev, type_data); 3722 default: 3723 return -EOPNOTSUPP; 3724 } 3725} 3726 3727static int cxgb_udp_tunnel_unset_port(struct net_device *netdev, 3728 unsigned int table, unsigned int entry, 3729 struct udp_tunnel_info *ti) 3730{ 3731 struct port_info *pi = netdev_priv(netdev); 3732 struct adapter *adapter = pi->adapter; 3733 u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; 3734 int ret = 0, i; 3735 3736 switch (ti->type) { 3737 case UDP_TUNNEL_TYPE_VXLAN: 3738 adapter->vxlan_port = 0; 3739 t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0); 3740 break; 3741 case UDP_TUNNEL_TYPE_GENEVE: 3742 adapter->geneve_port = 0; 3743 t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0); 3744 break; 3745 default: 3746 return -EINVAL; 3747 } 3748 3749 /* Matchall mac entries can be deleted only after all tunnel ports 3750 * are brought down or removed. 3751 */ 3752 if (!adapter->rawf_cnt) 3753 return 0; 3754 for_each_port(adapter, i) { 3755 pi = adap2pinfo(adapter, i); 3756 ret = t4_free_raw_mac_filt(adapter, pi->viid, 3757 match_all_mac, match_all_mac, 3758 adapter->rawf_start + pi->port_id, 3759 1, pi->port_id, false); 3760 if (ret < 0) { 3761 netdev_info(netdev, "Failed to free mac filter entry, for port %d\n", 3762 i); 3763 return ret; 3764 } 3765 } 3766 3767 return 0; 3768} 3769 3770static int cxgb_udp_tunnel_set_port(struct net_device *netdev, 3771 unsigned int table, unsigned int entry, 3772 struct udp_tunnel_info *ti) 3773{ 3774 struct port_info *pi = netdev_priv(netdev); 3775 struct adapter *adapter = pi->adapter; 3776 u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; 3777 int i, ret; 3778 3779 switch (ti->type) { 3780 case UDP_TUNNEL_TYPE_VXLAN: 3781 adapter->vxlan_port = ti->port; 3782 t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 3783 VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F); 3784 break; 3785 case UDP_TUNNEL_TYPE_GENEVE: 3786 adapter->geneve_port = ti->port; 3787 t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 3788 GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F); 3789 break; 3790 default: 3791 return -EINVAL; 3792 } 3793 3794 /* Create a 'match all' mac filter entry for inner mac, 3795 * if raw mac interface is supported. Once the linux kernel provides 3796 * driver entry points for adding/deleting the inner mac addresses, 3797 * we will remove this 'match all' entry and fallback to adding 3798 * exact match filters. 3799 */ 3800 for_each_port(adapter, i) { 3801 pi = adap2pinfo(adapter, i); 3802 3803 ret = t4_alloc_raw_mac_filt(adapter, pi->viid, 3804 match_all_mac, 3805 match_all_mac, 3806 adapter->rawf_start + pi->port_id, 3807 1, pi->port_id, false); 3808 if (ret < 0) { 3809 netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n", 3810 be16_to_cpu(ti->port)); 3811 return ret; 3812 } 3813 } 3814 3815 return 0; 3816} 3817 3818static const struct udp_tunnel_nic_info cxgb_udp_tunnels = { 3819 .set_port = cxgb_udp_tunnel_set_port, 3820 .unset_port = cxgb_udp_tunnel_unset_port, 3821 .tables = { 3822 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 3823 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 3824 }, 3825}; 3826 3827static netdev_features_t cxgb_features_check(struct sk_buff *skb, 3828 struct net_device *dev, 3829 netdev_features_t features) 3830{ 3831 struct port_info *pi = netdev_priv(dev); 3832 struct adapter *adapter = pi->adapter; 3833 3834 if (CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) 3835 return features; 3836 3837 /* Check if hw supports offload for this packet */ 3838 if (!skb->encapsulation || cxgb_encap_offload_supported(skb)) 3839 return features; 3840 3841 /* Offload is not supported for this encapsulated packet */ 3842 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3843} 3844 3845static netdev_features_t cxgb_fix_features(struct net_device *dev, 3846 netdev_features_t features) 3847{ 3848 /* Disable GRO, if RX_CSUM is disabled */ 3849 if (!(features & NETIF_F_RXCSUM)) 3850 features &= ~NETIF_F_GRO; 3851 3852 return features; 3853} 3854 3855static const struct net_device_ops cxgb4_netdev_ops = { 3856 .ndo_open = cxgb_open, 3857 .ndo_stop = cxgb_close, 3858 .ndo_start_xmit = t4_start_xmit, 3859 .ndo_select_queue = cxgb_select_queue, 3860 .ndo_get_stats64 = cxgb_get_stats, 3861 .ndo_set_rx_mode = cxgb_set_rxmode, 3862 .ndo_set_mac_address = cxgb_set_mac_addr, 3863 .ndo_set_features = cxgb_set_features, 3864 .ndo_validate_addr = eth_validate_addr, 3865 .ndo_eth_ioctl = cxgb_ioctl, 3866 .ndo_change_mtu = cxgb_change_mtu, 3867#ifdef CONFIG_NET_POLL_CONTROLLER 3868 .ndo_poll_controller = cxgb_netpoll, 3869#endif 3870#ifdef CONFIG_CHELSIO_T4_FCOE 3871 .ndo_fcoe_enable = cxgb_fcoe_enable, 3872 .ndo_fcoe_disable = cxgb_fcoe_disable, 3873#endif /* CONFIG_CHELSIO_T4_FCOE */ 3874 .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, 3875 .ndo_setup_tc = cxgb_setup_tc, 3876 .ndo_features_check = cxgb_features_check, 3877 .ndo_fix_features = cxgb_fix_features, 3878 .ndo_hwtstamp_get = cxgb_hwtstamp_get, 3879 .ndo_hwtstamp_set = cxgb_hwtstamp_set, 3880}; 3881 3882#ifdef CONFIG_PCI_IOV 3883static const struct net_device_ops cxgb4_mgmt_netdev_ops = { 3884 .ndo_open = cxgb4_mgmt_open, 3885 .ndo_set_vf_mac = cxgb4_mgmt_set_vf_mac, 3886 .ndo_get_vf_config = cxgb4_mgmt_get_vf_config, 3887 .ndo_set_vf_rate = cxgb4_mgmt_set_vf_rate, 3888 .ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id, 3889 .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan, 3890 .ndo_set_vf_link_state = cxgb4_mgmt_set_vf_link_state, 3891}; 3892 3893static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, 3894 struct ethtool_drvinfo *info) 3895{ 3896 struct adapter *adapter = netdev2adap(dev); 3897 3898 strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); 3899 strscpy(info->bus_info, pci_name(adapter->pdev), 3900 sizeof(info->bus_info)); 3901} 3902 3903static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { 3904 .get_drvinfo = cxgb4_mgmt_get_drvinfo, 3905}; 3906#endif 3907 3908static void notify_fatal_err(struct work_struct *work) 3909{ 3910 struct adapter *adap; 3911 3912 adap = container_of(work, struct adapter, fatal_err_notify_task); 3913 notify_ulds(adap, CXGB4_STATE_FATAL_ERROR); 3914} 3915 3916void t4_fatal_err(struct adapter *adap) 3917{ 3918 int port; 3919 3920 if (pci_channel_offline(adap->pdev)) 3921 return; 3922 3923 /* Disable the SGE since ULDs are going to free resources that 3924 * could be exposed to the adapter. RDMA MWs for example... 3925 */ 3926 t4_shutdown_adapter(adap); 3927 for_each_port(adap, port) { 3928 struct net_device *dev = adap->port[port]; 3929 3930 /* If we get here in very early initialization the network 3931 * devices may not have been set up yet. 3932 */ 3933 if (!dev) 3934 continue; 3935 3936 netif_tx_stop_all_queues(dev); 3937 netif_carrier_off(dev); 3938 } 3939 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 3940 queue_work(adap->workq, &adap->fatal_err_notify_task); 3941} 3942 3943static void setup_memwin(struct adapter *adap) 3944{ 3945 u32 nic_win_base = t4_get_util_window(adap); 3946 3947 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); 3948} 3949 3950static void setup_memwin_rdma(struct adapter *adap) 3951{ 3952 if (adap->vres.ocq.size) { 3953 u32 start; 3954 unsigned int sz_kb; 3955 3956 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 3957 start &= PCI_BASE_ADDRESS_MEM_MASK; 3958 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 3959 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 3960 t4_write_reg(adap, 3961 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 3962 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 3963 t4_write_reg(adap, 3964 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 3965 adap->vres.ocq.start); 3966 t4_read_reg(adap, 3967 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 3968 } 3969} 3970 3971/* HMA Definitions */ 3972 3973/* The maximum number of address that can be send in a single FW cmd */ 3974#define HMA_MAX_ADDR_IN_CMD 5 3975 3976#define HMA_PAGE_SIZE PAGE_SIZE 3977 3978#define HMA_MAX_NO_FW_ADDRESS (16 << 10) /* FW supports 16K addresses */ 3979 3980#define HMA_PAGE_ORDER \ 3981 ((HMA_PAGE_SIZE < HMA_MAX_NO_FW_ADDRESS) ? \ 3982 ilog2(HMA_MAX_NO_FW_ADDRESS / HMA_PAGE_SIZE) : 0) 3983 3984/* The minimum and maximum possible HMA sizes that can be specified in the FW 3985 * configuration(in units of MB). 3986 */ 3987#define HMA_MIN_TOTAL_SIZE 1 3988#define HMA_MAX_TOTAL_SIZE \ 3989 (((HMA_PAGE_SIZE << HMA_PAGE_ORDER) * \ 3990 HMA_MAX_NO_FW_ADDRESS) >> 20) 3991 3992static void adap_free_hma_mem(struct adapter *adapter) 3993{ 3994 struct scatterlist *iter; 3995 struct page *page; 3996 int i; 3997 3998 if (!adapter->hma.sgt) 3999 return; 4000 4001 if (adapter->hma.flags & HMA_DMA_MAPPED_FLAG) { 4002 dma_unmap_sg(adapter->pdev_dev, adapter->hma.sgt->sgl, 4003 adapter->hma.sgt->nents, DMA_BIDIRECTIONAL); 4004 adapter->hma.flags &= ~HMA_DMA_MAPPED_FLAG; 4005 } 4006 4007 for_each_sg(adapter->hma.sgt->sgl, iter, 4008 adapter->hma.sgt->orig_nents, i) { 4009 page = sg_page(iter); 4010 if (page) 4011 __free_pages(page, HMA_PAGE_ORDER); 4012 } 4013 4014 kfree(adapter->hma.phy_addr); 4015 sg_free_table(adapter->hma.sgt); 4016 kfree(adapter->hma.sgt); 4017 adapter->hma.sgt = NULL; 4018} 4019 4020static int adap_config_hma(struct adapter *adapter) 4021{ 4022 struct scatterlist *sgl, *iter; 4023 struct sg_table *sgt; 4024 struct page *newpage; 4025 unsigned int i, j, k; 4026 u32 param, hma_size; 4027 unsigned int ncmds; 4028 size_t page_size; 4029 u32 page_order; 4030 int node, ret; 4031 4032 /* HMA is supported only for T6+ cards. 4033 * Avoid initializing HMA in kdump kernels. 4034 */ 4035 if (is_kdump_kernel() || 4036 CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) 4037 return 0; 4038 4039 /* Get the HMA region size required by fw */ 4040 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4041 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HMA_SIZE)); 4042 ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 4043 1, ¶m, &hma_size); 4044 /* An error means card has its own memory or HMA is not supported by 4045 * the firmware. Return without any errors. 4046 */ 4047 if (ret || !hma_size) 4048 return 0; 4049 4050 if (hma_size < HMA_MIN_TOTAL_SIZE || 4051 hma_size > HMA_MAX_TOTAL_SIZE) { 4052 dev_err(adapter->pdev_dev, 4053 "HMA size %uMB beyond bounds(%u-%lu)MB\n", 4054 hma_size, HMA_MIN_TOTAL_SIZE, HMA_MAX_TOTAL_SIZE); 4055 return -EINVAL; 4056 } 4057 4058 page_size = HMA_PAGE_SIZE; 4059 page_order = HMA_PAGE_ORDER; 4060 adapter->hma.sgt = kzalloc(sizeof(*adapter->hma.sgt), GFP_KERNEL); 4061 if (unlikely(!adapter->hma.sgt)) { 4062 dev_err(adapter->pdev_dev, "HMA SG table allocation failed\n"); 4063 return -ENOMEM; 4064 } 4065 sgt = adapter->hma.sgt; 4066 /* FW returned value will be in MB's 4067 */ 4068 sgt->orig_nents = (hma_size << 20) / (page_size << page_order); 4069 if (sg_alloc_table(sgt, sgt->orig_nents, GFP_KERNEL)) { 4070 dev_err(adapter->pdev_dev, "HMA SGL allocation failed\n"); 4071 kfree(adapter->hma.sgt); 4072 adapter->hma.sgt = NULL; 4073 return -ENOMEM; 4074 } 4075 4076 sgl = adapter->hma.sgt->sgl; 4077 node = dev_to_node(adapter->pdev_dev); 4078 for_each_sg(sgl, iter, sgt->orig_nents, i) { 4079 newpage = alloc_pages_node(node, __GFP_NOWARN | GFP_KERNEL | 4080 __GFP_ZERO, page_order); 4081 if (!newpage) { 4082 dev_err(adapter->pdev_dev, 4083 "Not enough memory for HMA page allocation\n"); 4084 ret = -ENOMEM; 4085 goto free_hma; 4086 } 4087 sg_set_page(iter, newpage, page_size << page_order, 0); 4088 } 4089 4090 sgt->nents = dma_map_sg(adapter->pdev_dev, sgl, sgt->orig_nents, 4091 DMA_BIDIRECTIONAL); 4092 if (!sgt->nents) { 4093 dev_err(adapter->pdev_dev, 4094 "Not enough memory for HMA DMA mapping"); 4095 ret = -ENOMEM; 4096 goto free_hma; 4097 } 4098 adapter->hma.flags |= HMA_DMA_MAPPED_FLAG; 4099 4100 adapter->hma.phy_addr = kcalloc(sgt->nents, sizeof(dma_addr_t), 4101 GFP_KERNEL); 4102 if (unlikely(!adapter->hma.phy_addr)) 4103 goto free_hma; 4104 4105 for_each_sg(sgl, iter, sgt->nents, i) { 4106 newpage = sg_page(iter); 4107 adapter->hma.phy_addr[i] = sg_dma_address(iter); 4108 } 4109 4110 ncmds = DIV_ROUND_UP(sgt->nents, HMA_MAX_ADDR_IN_CMD); 4111 /* Pass on the addresses to firmware */ 4112 for (i = 0, k = 0; i < ncmds; i++, k += HMA_MAX_ADDR_IN_CMD) { 4113 struct fw_hma_cmd hma_cmd; 4114 u8 naddr = HMA_MAX_ADDR_IN_CMD; 4115 u8 soc = 0, eoc = 0; 4116 u8 hma_mode = 1; /* Presently we support only Page table mode */ 4117 4118 soc = (i == 0) ? 1 : 0; 4119 eoc = (i == ncmds - 1) ? 1 : 0; 4120 4121 /* For last cmd, set naddr corresponding to remaining 4122 * addresses 4123 */ 4124 if (i == ncmds - 1) { 4125 naddr = sgt->nents % HMA_MAX_ADDR_IN_CMD; 4126 naddr = naddr ? naddr : HMA_MAX_ADDR_IN_CMD; 4127 } 4128 memset(&hma_cmd, 0, sizeof(hma_cmd)); 4129 hma_cmd.op_pkd = htonl(FW_CMD_OP_V(FW_HMA_CMD) | 4130 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 4131 hma_cmd.retval_len16 = htonl(FW_LEN16(hma_cmd)); 4132 4133 hma_cmd.mode_to_pcie_params = 4134 htonl(FW_HMA_CMD_MODE_V(hma_mode) | 4135 FW_HMA_CMD_SOC_V(soc) | FW_HMA_CMD_EOC_V(eoc)); 4136 4137 /* HMA cmd size specified in MB's */ 4138 hma_cmd.naddr_size = 4139 htonl(FW_HMA_CMD_SIZE_V(hma_size) | 4140 FW_HMA_CMD_NADDR_V(naddr)); 4141 4142 /* Total Page size specified in units of 4K */ 4143 hma_cmd.addr_size_pkd = 4144 htonl(FW_HMA_CMD_ADDR_SIZE_V 4145 ((page_size << page_order) >> 12)); 4146 4147 /* Fill the 5 addresses */ 4148 for (j = 0; j < naddr; j++) { 4149 hma_cmd.phy_address[j] = 4150 cpu_to_be64(adapter->hma.phy_addr[j + k]); 4151 } 4152 ret = t4_wr_mbox(adapter, adapter->mbox, &hma_cmd, 4153 sizeof(hma_cmd), &hma_cmd); 4154 if (ret) { 4155 dev_err(adapter->pdev_dev, 4156 "HMA FW command failed with err %d\n", ret); 4157 goto free_hma; 4158 } 4159 } 4160 4161 if (!ret) 4162 dev_info(adapter->pdev_dev, 4163 "Reserved %uMB host memory for HMA\n", hma_size); 4164 return ret; 4165 4166free_hma: 4167 adap_free_hma_mem(adapter); 4168 return ret; 4169} 4170 4171static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 4172{ 4173 u32 v; 4174 int ret; 4175 4176 /* Now that we've successfully configured and initialized the adapter 4177 * can ask the Firmware what resources it has provisioned for us. 4178 */ 4179 ret = t4_get_pfres(adap); 4180 if (ret) { 4181 dev_err(adap->pdev_dev, 4182 "Unable to retrieve resource provisioning information\n"); 4183 return ret; 4184 } 4185 4186 /* get device capabilities */ 4187 memset(c, 0, sizeof(*c)); 4188 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4189 FW_CMD_REQUEST_F | FW_CMD_READ_F); 4190 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 4191 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); 4192 if (ret < 0) 4193 return ret; 4194 4195 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4196 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 4197 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); 4198 if (ret < 0) 4199 return ret; 4200 4201 ret = t4_config_glbl_rss(adap, adap->pf, 4202 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 4203 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 4204 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 4205 if (ret < 0) 4206 return ret; 4207 4208 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, 4209 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, 4210 FW_CMD_CAP_PF); 4211 if (ret < 0) 4212 return ret; 4213 4214 t4_sge_init(adap); 4215 4216 /* tweak some settings */ 4217 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 4218 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 4219 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 4220 v = t4_read_reg(adap, TP_PIO_DATA_A); 4221 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 4222 4223 /* first 4 Tx modulation queues point to consecutive Tx channels */ 4224 adap->params.tp.tx_modq_map = 0xE4; 4225 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 4226 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 4227 4228 /* associate each Tx modulation queue with consecutive Tx channels */ 4229 v = 0x84218421; 4230 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4231 &v, 1, TP_TX_SCHED_HDR_A); 4232 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4233 &v, 1, TP_TX_SCHED_FIFO_A); 4234 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4235 &v, 1, TP_TX_SCHED_PCMD_A); 4236 4237#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 4238 if (is_offload(adap)) { 4239 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 4240 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4241 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4242 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4243 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4244 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 4245 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4246 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4247 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4248 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4249 } 4250 4251 /* get basic stuff going */ 4252 return t4_early_init(adap, adap->pf); 4253} 4254 4255/* 4256 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 4257 */ 4258#define MAX_ATIDS 8192U 4259 4260/* 4261 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 4262 * 4263 * If the firmware we're dealing with has Configuration File support, then 4264 * we use that to perform all configuration 4265 */ 4266 4267/* 4268 * Tweak configuration based on module parameters, etc. Most of these have 4269 * defaults assigned to them by Firmware Configuration Files (if we're using 4270 * them) but need to be explicitly set if we're using hard-coded 4271 * initialization. But even in the case of using Firmware Configuration 4272 * Files, we'd like to expose the ability to change these via module 4273 * parameters so these are essentially common tweaks/settings for 4274 * Configuration Files and hard-coded initialization ... 4275 */ 4276static int adap_init0_tweaks(struct adapter *adapter) 4277{ 4278 /* 4279 * Fix up various Host-Dependent Parameters like Page Size, Cache 4280 * Line Size, etc. The firmware default is for a 4KB Page Size and 4281 * 64B Cache Line Size ... 4282 */ 4283 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 4284 4285 /* 4286 * Process module parameters which affect early initialization. 4287 */ 4288 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 4289 dev_err(&adapter->pdev->dev, 4290 "Ignoring illegal rx_dma_offset=%d, using 2\n", 4291 rx_dma_offset); 4292 rx_dma_offset = 2; 4293 } 4294 t4_set_reg_field(adapter, SGE_CONTROL_A, 4295 PKTSHIFT_V(PKTSHIFT_M), 4296 PKTSHIFT_V(rx_dma_offset)); 4297 4298 /* 4299 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 4300 * adds the pseudo header itself. 4301 */ 4302 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 4303 CSUM_HAS_PSEUDO_HDR_F, 0); 4304 4305 return 0; 4306} 4307 4308/* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips 4309 * unto themselves and they contain their own firmware to perform their 4310 * tasks ... 4311 */ 4312static int phy_aq1202_version(const u8 *phy_fw_data, 4313 size_t phy_fw_size) 4314{ 4315 int offset; 4316 4317 /* At offset 0x8 you're looking for the primary image's 4318 * starting offset which is 3 Bytes wide 4319 * 4320 * At offset 0xa of the primary image, you look for the offset 4321 * of the DRAM segment which is 3 Bytes wide. 4322 * 4323 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes 4324 * wide 4325 */ 4326 #define be16(__p) (((__p)[0] << 8) | (__p)[1]) 4327 #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) 4328 #define le24(__p) (le16(__p) | ((__p)[2] << 16)) 4329 4330 offset = le24(phy_fw_data + 0x8) << 12; 4331 offset = le24(phy_fw_data + offset + 0xa); 4332 return be16(phy_fw_data + offset + 0x27e); 4333 4334 #undef be16 4335 #undef le16 4336 #undef le24 4337} 4338 4339static struct info_10gbt_phy_fw { 4340 unsigned int phy_fw_id; /* PCI Device ID */ 4341 char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ 4342 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); 4343 int phy_flash; /* Has FLASH for PHY Firmware */ 4344} phy_info_array[] = { 4345 { 4346 PHY_AQ1202_DEVICEID, 4347 PHY_AQ1202_FIRMWARE, 4348 phy_aq1202_version, 4349 1, 4350 }, 4351 { 4352 PHY_BCM84834_DEVICEID, 4353 PHY_BCM84834_FIRMWARE, 4354 NULL, 4355 0, 4356 }, 4357 { 0, NULL, NULL }, 4358}; 4359 4360static struct info_10gbt_phy_fw *find_phy_info(int devid) 4361{ 4362 int i; 4363 4364 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { 4365 if (phy_info_array[i].phy_fw_id == devid) 4366 return &phy_info_array[i]; 4367 } 4368 return NULL; 4369} 4370 4371/* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to 4372 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error 4373 * we return a negative error number. If we transfer new firmware we return 1 4374 * (from t4_load_phy_fw()). If we don't do anything we return 0. 4375 */ 4376static int adap_init0_phy(struct adapter *adap) 4377{ 4378 const struct firmware *phyf; 4379 int ret; 4380 struct info_10gbt_phy_fw *phy_info; 4381 4382 /* Use the device ID to determine which PHY file to flash. 4383 */ 4384 phy_info = find_phy_info(adap->pdev->device); 4385 if (!phy_info) { 4386 dev_warn(adap->pdev_dev, 4387 "No PHY Firmware file found for this PHY\n"); 4388 return -EOPNOTSUPP; 4389 } 4390 4391 /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then 4392 * use that. The adapter firmware provides us with a memory buffer 4393 * where we can load a PHY firmware file from the host if we want to 4394 * override the PHY firmware File in flash. 4395 */ 4396 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, 4397 adap->pdev_dev); 4398 if (ret < 0) { 4399 /* For adapters without FLASH attached to PHY for their 4400 * firmware, it's obviously a fatal error if we can't get the 4401 * firmware to the adapter. For adapters with PHY firmware 4402 * FLASH storage, it's worth a warning if we can't find the 4403 * PHY Firmware but we'll neuter the error ... 4404 */ 4405 dev_err(adap->pdev_dev, "unable to find PHY Firmware image " 4406 "/lib/firmware/%s, error %d\n", 4407 phy_info->phy_fw_file, -ret); 4408 if (phy_info->phy_flash) { 4409 int cur_phy_fw_ver = 0; 4410 4411 t4_phy_fw_ver(adap, &cur_phy_fw_ver); 4412 dev_warn(adap->pdev_dev, "continuing with, on-adapter " 4413 "FLASH copy, version %#x\n", cur_phy_fw_ver); 4414 ret = 0; 4415 } 4416 4417 return ret; 4418 } 4419 4420 /* Load PHY Firmware onto adapter. 4421 */ 4422 ret = t4_load_phy_fw(adap, MEMWIN_NIC, phy_info->phy_fw_version, 4423 (u8 *)phyf->data, phyf->size); 4424 if (ret < 0) 4425 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", 4426 -ret); 4427 else if (ret > 0) { 4428 int new_phy_fw_ver = 0; 4429 4430 if (phy_info->phy_fw_version) 4431 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, 4432 phyf->size); 4433 dev_info(adap->pdev_dev, "Successfully transferred PHY " 4434 "Firmware /lib/firmware/%s, version %#x\n", 4435 phy_info->phy_fw_file, new_phy_fw_ver); 4436 } 4437 4438 release_firmware(phyf); 4439 4440 return ret; 4441} 4442 4443/* 4444 * Attempt to initialize the adapter via a Firmware Configuration File. 4445 */ 4446static int adap_init0_config(struct adapter *adapter, int reset) 4447{ 4448 char *fw_config_file, fw_config_file_path[256]; 4449 u32 finiver, finicsum, cfcsum, param, val; 4450 struct fw_caps_config_cmd caps_cmd; 4451 unsigned long mtype = 0, maddr = 0; 4452 const struct firmware *cf; 4453 char *config_name = NULL; 4454 int config_issued = 0; 4455 int ret; 4456 4457 /* 4458 * Reset device if necessary. 4459 */ 4460 if (reset) { 4461 ret = t4_fw_reset(adapter, adapter->mbox, 4462 PIORSTMODE_F | PIORST_F); 4463 if (ret < 0) 4464 goto bye; 4465 } 4466 4467 /* If this is a 10Gb/s-BT adapter make sure the chip-external 4468 * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs 4469 * to be performed after any global adapter RESET above since some 4470 * PHYs only have local RAM copies of the PHY firmware. 4471 */ 4472 if (is_10gbt_device(adapter->pdev->device)) { 4473 ret = adap_init0_phy(adapter); 4474 if (ret < 0) 4475 goto bye; 4476 } 4477 /* 4478 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 4479 * then use that. Otherwise, use the configuration file stored 4480 * in the adapter flash ... 4481 */ 4482 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 4483 case CHELSIO_T4: 4484 fw_config_file = FW4_CFNAME; 4485 break; 4486 case CHELSIO_T5: 4487 fw_config_file = FW5_CFNAME; 4488 break; 4489 case CHELSIO_T6: 4490 fw_config_file = FW6_CFNAME; 4491 break; 4492 default: 4493 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 4494 adapter->pdev->device); 4495 ret = -EINVAL; 4496 goto bye; 4497 } 4498 4499 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 4500 if (ret < 0) { 4501 config_name = "On FLASH"; 4502 mtype = FW_MEMTYPE_CF_FLASH; 4503 maddr = t4_flash_cfg_addr(adapter); 4504 } else { 4505 u32 params[7], val[7]; 4506 4507 sprintf(fw_config_file_path, 4508 "/lib/firmware/%s", fw_config_file); 4509 config_name = fw_config_file_path; 4510 4511 if (cf->size >= FLASH_CFG_MAX_SIZE) 4512 ret = -ENOMEM; 4513 else { 4514 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4515 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 4516 ret = t4_query_params(adapter, adapter->mbox, 4517 adapter->pf, 0, 1, params, val); 4518 if (ret == 0) { 4519 /* 4520 * For t4_memory_rw() below addresses and 4521 * sizes have to be in terms of multiples of 4 4522 * bytes. So, if the Configuration File isn't 4523 * a multiple of 4 bytes in length we'll have 4524 * to write that out separately since we can't 4525 * guarantee that the bytes following the 4526 * residual byte in the buffer returned by 4527 * request_firmware() are zeroed out ... 4528 */ 4529 size_t resid = cf->size & 0x3; 4530 size_t size = cf->size & ~0x3; 4531 __be32 *data = (__be32 *)cf->data; 4532 4533 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 4534 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 4535 4536 spin_lock(&adapter->win0_lock); 4537 ret = t4_memory_rw(adapter, 0, mtype, maddr, 4538 size, data, T4_MEMORY_WRITE); 4539 if (ret == 0 && resid != 0) { 4540 union { 4541 __be32 word; 4542 char buf[4]; 4543 } last; 4544 int i; 4545 4546 last.word = data[size >> 2]; 4547 for (i = resid; i < 4; i++) 4548 last.buf[i] = 0; 4549 ret = t4_memory_rw(adapter, 0, mtype, 4550 maddr + size, 4551 4, &last.word, 4552 T4_MEMORY_WRITE); 4553 } 4554 spin_unlock(&adapter->win0_lock); 4555 } 4556 } 4557 4558 release_firmware(cf); 4559 if (ret) 4560 goto bye; 4561 } 4562 4563 val = 0; 4564 4565 /* Ofld + Hash filter is supported. Older fw will fail this request and 4566 * it is fine. 4567 */ 4568 param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4569 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HASHFILTER_WITH_OFLD)); 4570 ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0, 4571 1, ¶m, &val); 4572 4573 /* FW doesn't know about Hash filter + ofld support, 4574 * it's not a problem, don't return an error. 4575 */ 4576 if (ret < 0) { 4577 dev_warn(adapter->pdev_dev, 4578 "Hash filter with ofld is not supported by FW\n"); 4579 } 4580 4581 /* 4582 * Issue a Capability Configuration command to the firmware to get it 4583 * to parse the Configuration File. We don't use t4_fw_config_file() 4584 * because we want the ability to modify various features after we've 4585 * processed the configuration file ... 4586 */ 4587 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4588 caps_cmd.op_to_write = 4589 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4590 FW_CMD_REQUEST_F | 4591 FW_CMD_READ_F); 4592 caps_cmd.cfvalid_to_len16 = 4593 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 4594 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 4595 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 4596 FW_LEN16(caps_cmd)); 4597 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 4598 &caps_cmd); 4599 4600 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 4601 * Configuration File in FLASH), our last gasp effort is to use the 4602 * Firmware Configuration File which is embedded in the firmware. A 4603 * very few early versions of the firmware didn't have one embedded 4604 * but we can ignore those. 4605 */ 4606 if (ret == -ENOENT) { 4607 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4608 caps_cmd.op_to_write = 4609 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4610 FW_CMD_REQUEST_F | 4611 FW_CMD_READ_F); 4612 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4613 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 4614 sizeof(caps_cmd), &caps_cmd); 4615 config_name = "Firmware Default"; 4616 } 4617 4618 config_issued = 1; 4619 if (ret < 0) 4620 goto bye; 4621 4622 finiver = ntohl(caps_cmd.finiver); 4623 finicsum = ntohl(caps_cmd.finicsum); 4624 cfcsum = ntohl(caps_cmd.cfcsum); 4625 if (finicsum != cfcsum) 4626 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 4627 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 4628 finicsum, cfcsum); 4629 4630 /* 4631 * And now tell the firmware to use the configuration we just loaded. 4632 */ 4633 caps_cmd.op_to_write = 4634 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4635 FW_CMD_REQUEST_F | 4636 FW_CMD_WRITE_F); 4637 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4638 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 4639 NULL); 4640 if (ret < 0) 4641 goto bye; 4642 4643 /* 4644 * Tweak configuration based on system architecture, module 4645 * parameters, etc. 4646 */ 4647 ret = adap_init0_tweaks(adapter); 4648 if (ret < 0) 4649 goto bye; 4650 4651 /* We will proceed even if HMA init fails. */ 4652 ret = adap_config_hma(adapter); 4653 if (ret) 4654 dev_err(adapter->pdev_dev, 4655 "HMA configuration failed with error %d\n", ret); 4656 4657 if (is_t6(adapter->params.chip)) { 4658 adap_config_hpfilter(adapter); 4659 ret = setup_ppod_edram(adapter); 4660 if (!ret) 4661 dev_info(adapter->pdev_dev, "Successfully enabled " 4662 "ppod edram feature\n"); 4663 } 4664 4665 /* 4666 * And finally tell the firmware to initialize itself using the 4667 * parameters from the Configuration File. 4668 */ 4669 ret = t4_fw_initialize(adapter, adapter->mbox); 4670 if (ret < 0) 4671 goto bye; 4672 4673 /* Emit Firmware Configuration File information and return 4674 * successfully. 4675 */ 4676 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 4677 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 4678 config_name, finiver, cfcsum); 4679 return 0; 4680 4681 /* 4682 * Something bad happened. Return the error ... (If the "error" 4683 * is that there's no Configuration File on the adapter we don't 4684 * want to issue a warning since this is fairly common.) 4685 */ 4686bye: 4687 if (config_issued && ret != -ENOENT) 4688 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 4689 config_name, -ret); 4690 return ret; 4691} 4692 4693static struct fw_info fw_info_array[] = { 4694 { 4695 .chip = CHELSIO_T4, 4696 .fs_name = FW4_CFNAME, 4697 .fw_mod_name = FW4_FNAME, 4698 .fw_hdr = { 4699 .chip = FW_HDR_CHIP_T4, 4700 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 4701 .intfver_nic = FW_INTFVER(T4, NIC), 4702 .intfver_vnic = FW_INTFVER(T4, VNIC), 4703 .intfver_ri = FW_INTFVER(T4, RI), 4704 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 4705 .intfver_fcoe = FW_INTFVER(T4, FCOE), 4706 }, 4707 }, { 4708 .chip = CHELSIO_T5, 4709 .fs_name = FW5_CFNAME, 4710 .fw_mod_name = FW5_FNAME, 4711 .fw_hdr = { 4712 .chip = FW_HDR_CHIP_T5, 4713 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 4714 .intfver_nic = FW_INTFVER(T5, NIC), 4715 .intfver_vnic = FW_INTFVER(T5, VNIC), 4716 .intfver_ri = FW_INTFVER(T5, RI), 4717 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 4718 .intfver_fcoe = FW_INTFVER(T5, FCOE), 4719 }, 4720 }, { 4721 .chip = CHELSIO_T6, 4722 .fs_name = FW6_CFNAME, 4723 .fw_mod_name = FW6_FNAME, 4724 .fw_hdr = { 4725 .chip = FW_HDR_CHIP_T6, 4726 .fw_ver = __cpu_to_be32(FW_VERSION(T6)), 4727 .intfver_nic = FW_INTFVER(T6, NIC), 4728 .intfver_vnic = FW_INTFVER(T6, VNIC), 4729 .intfver_ofld = FW_INTFVER(T6, OFLD), 4730 .intfver_ri = FW_INTFVER(T6, RI), 4731 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 4732 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 4733 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 4734 .intfver_fcoe = FW_INTFVER(T6, FCOE), 4735 }, 4736 } 4737 4738}; 4739 4740static struct fw_info *find_fw_info(int chip) 4741{ 4742 int i; 4743 4744 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 4745 if (fw_info_array[i].chip == chip) 4746 return &fw_info_array[i]; 4747 } 4748 return NULL; 4749} 4750 4751/* 4752 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 4753 */ 4754static int adap_init0(struct adapter *adap, int vpd_skip) 4755{ 4756 struct fw_caps_config_cmd caps_cmd; 4757 u32 params[7], val[7]; 4758 enum dev_state state; 4759 u32 v, port_vec; 4760 int reset = 1; 4761 int ret; 4762 4763 /* Grab Firmware Device Log parameters as early as possible so we have 4764 * access to it for debugging, etc. 4765 */ 4766 ret = t4_init_devlog_params(adap); 4767 if (ret < 0) 4768 return ret; 4769 4770 /* Contact FW, advertising Master capability */ 4771 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, 4772 is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state); 4773 if (ret < 0) { 4774 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 4775 ret); 4776 return ret; 4777 } 4778 if (ret == adap->mbox) 4779 adap->flags |= CXGB4_MASTER_PF; 4780 4781 /* 4782 * If we're the Master PF Driver and the device is uninitialized, 4783 * then let's consider upgrading the firmware ... (We always want 4784 * to check the firmware version number in order to A. get it for 4785 * later reporting and B. to warn if the currently loaded firmware 4786 * is excessively mismatched relative to the driver.) 4787 */ 4788 4789 t4_get_version_info(adap); 4790 ret = t4_check_fw_version(adap); 4791 /* If firmware is too old (not supported by driver) force an update. */ 4792 if (ret) 4793 state = DEV_STATE_UNINIT; 4794 if ((adap->flags & CXGB4_MASTER_PF) && state != DEV_STATE_INIT) { 4795 struct fw_info *fw_info; 4796 struct fw_hdr *card_fw; 4797 const struct firmware *fw; 4798 const u8 *fw_data = NULL; 4799 unsigned int fw_size = 0; 4800 4801 /* This is the firmware whose headers the driver was compiled 4802 * against 4803 */ 4804 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 4805 if (fw_info == NULL) { 4806 dev_err(adap->pdev_dev, 4807 "unable to get firmware info for chip %d.\n", 4808 CHELSIO_CHIP_VERSION(adap->params.chip)); 4809 return -EINVAL; 4810 } 4811 4812 /* allocate memory to read the header of the firmware on the 4813 * card 4814 */ 4815 card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL); 4816 if (!card_fw) { 4817 ret = -ENOMEM; 4818 goto bye; 4819 } 4820 4821 /* Get FW from /lib/firmware/ */ 4822 ret = request_firmware(&fw, fw_info->fw_mod_name, 4823 adap->pdev_dev); 4824 if (ret < 0) { 4825 dev_err(adap->pdev_dev, 4826 "unable to load firmware image %s, error %d\n", 4827 fw_info->fw_mod_name, ret); 4828 } else { 4829 fw_data = fw->data; 4830 fw_size = fw->size; 4831 } 4832 4833 /* upgrade FW logic */ 4834 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 4835 state, &reset); 4836 4837 /* Cleaning up */ 4838 release_firmware(fw); 4839 kvfree(card_fw); 4840 4841 if (ret < 0) 4842 goto bye; 4843 } 4844 4845 /* If the firmware is initialized already, emit a simply note to that 4846 * effect. Otherwise, it's time to try initializing the adapter. 4847 */ 4848 if (state == DEV_STATE_INIT) { 4849 ret = adap_config_hma(adap); 4850 if (ret) 4851 dev_err(adap->pdev_dev, 4852 "HMA configuration failed with error %d\n", 4853 ret); 4854 dev_info(adap->pdev_dev, "Coming up as %s: "\ 4855 "Adapter already initialized\n", 4856 adap->flags & CXGB4_MASTER_PF ? "MASTER" : "SLAVE"); 4857 } else { 4858 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 4859 "Initializing adapter\n"); 4860 4861 /* Find out whether we're dealing with a version of the 4862 * firmware which has configuration file support. 4863 */ 4864 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4865 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 4866 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, 4867 params, val); 4868 4869 /* If the firmware doesn't support Configuration Files, 4870 * return an error. 4871 */ 4872 if (ret < 0) { 4873 dev_err(adap->pdev_dev, "firmware doesn't support " 4874 "Firmware Configuration Files\n"); 4875 goto bye; 4876 } 4877 4878 /* The firmware provides us with a memory buffer where we can 4879 * load a Configuration File from the host if we want to 4880 * override the Configuration File in flash. 4881 */ 4882 ret = adap_init0_config(adap, reset); 4883 if (ret == -ENOENT) { 4884 dev_err(adap->pdev_dev, "no Configuration File " 4885 "present on adapter.\n"); 4886 goto bye; 4887 } 4888 if (ret < 0) { 4889 dev_err(adap->pdev_dev, "could not initialize " 4890 "adapter, error %d\n", -ret); 4891 goto bye; 4892 } 4893 } 4894 4895 /* Now that we've successfully configured and initialized the adapter 4896 * (or found it already initialized), we can ask the Firmware what 4897 * resources it has provisioned for us. 4898 */ 4899 ret = t4_get_pfres(adap); 4900 if (ret) { 4901 dev_err(adap->pdev_dev, 4902 "Unable to retrieve resource provisioning information\n"); 4903 goto bye; 4904 } 4905 4906 /* Grab VPD parameters. This should be done after we establish a 4907 * connection to the firmware since some of the VPD parameters 4908 * (notably the Core Clock frequency) are retrieved via requests to 4909 * the firmware. On the other hand, we need these fairly early on 4910 * so we do this right after getting ahold of the firmware. 4911 * 4912 * We need to do this after initializing the adapter because someone 4913 * could have FLASHed a new VPD which won't be read by the firmware 4914 * until we do the RESET ... 4915 */ 4916 if (!vpd_skip) { 4917 ret = t4_get_vpd_params(adap, &adap->params.vpd); 4918 if (ret < 0) 4919 goto bye; 4920 } 4921 4922 /* Find out what ports are available to us. Note that we need to do 4923 * this before calling adap_init0_no_config() since it needs nports 4924 * and portvec ... 4925 */ 4926 v = 4927 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4928 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 4929 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); 4930 if (ret < 0) 4931 goto bye; 4932 4933 adap->params.nports = hweight32(port_vec); 4934 adap->params.portvec = port_vec; 4935 4936 /* Give the SGE code a chance to pull in anything that it needs ... 4937 * Note that this must be called after we retrieve our VPD parameters 4938 * in order to know how to convert core ticks to seconds, etc. 4939 */ 4940 ret = t4_sge_init(adap); 4941 if (ret < 0) 4942 goto bye; 4943 4944 /* Grab the SGE Doorbell Queue Timer values. If successful, that 4945 * indicates that the Firmware and Hardware support this. 4946 */ 4947 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4948 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK)); 4949 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4950 1, params, val); 4951 4952 if (!ret) { 4953 adap->sge.dbqtimer_tick = val[0]; 4954 ret = t4_read_sge_dbqtimers(adap, 4955 ARRAY_SIZE(adap->sge.dbqtimer_val), 4956 adap->sge.dbqtimer_val); 4957 } 4958 4959 if (!ret) 4960 adap->flags |= CXGB4_SGE_DBQ_TIMER; 4961 4962 if (is_bypass_device(adap->pdev->device)) 4963 adap->params.bypass = 1; 4964 4965 /* 4966 * Grab some of our basic fundamental operating parameters. 4967 */ 4968 params[0] = FW_PARAM_PFVF(EQ_START); 4969 params[1] = FW_PARAM_PFVF(L2T_START); 4970 params[2] = FW_PARAM_PFVF(L2T_END); 4971 params[3] = FW_PARAM_PFVF(FILTER_START); 4972 params[4] = FW_PARAM_PFVF(FILTER_END); 4973 params[5] = FW_PARAM_PFVF(IQFLINT_START); 4974 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); 4975 if (ret < 0) 4976 goto bye; 4977 adap->sge.egr_start = val[0]; 4978 adap->l2t_start = val[1]; 4979 adap->l2t_end = val[2]; 4980 adap->tids.ftid_base = val[3]; 4981 adap->tids.nftids = val[4] - val[3] + 1; 4982 adap->sge.ingr_start = val[5]; 4983 4984 if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) { 4985 params[0] = FW_PARAM_PFVF(HPFILTER_START); 4986 params[1] = FW_PARAM_PFVF(HPFILTER_END); 4987 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4988 params, val); 4989 if (ret < 0) 4990 goto bye; 4991 4992 adap->tids.hpftid_base = val[0]; 4993 adap->tids.nhpftids = val[1] - val[0] + 1; 4994 4995 /* Read the raw mps entries. In T6, the last 2 tcam entries 4996 * are reserved for raw mac addresses (rawf = 2, one per port). 4997 */ 4998 params[0] = FW_PARAM_PFVF(RAWF_START); 4999 params[1] = FW_PARAM_PFVF(RAWF_END); 5000 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 5001 params, val); 5002 if (ret == 0) { 5003 adap->rawf_start = val[0]; 5004 adap->rawf_cnt = val[1] - val[0] + 1; 5005 } 5006 5007 adap->tids.tid_base = 5008 t4_read_reg(adap, LE_DB_ACTIVE_TABLE_START_INDEX_A); 5009 } 5010 5011 /* qids (ingress/egress) returned from firmware can be anywhere 5012 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. 5013 * Hence driver needs to allocate memory for this range to 5014 * store the queue info. Get the highest IQFLINT/EQ index returned 5015 * in FW_EQ_*_CMD.alloc command. 5016 */ 5017 params[0] = FW_PARAM_PFVF(EQ_END); 5018 params[1] = FW_PARAM_PFVF(IQFLINT_END); 5019 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 5020 if (ret < 0) 5021 goto bye; 5022 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; 5023 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; 5024 5025 adap->sge.egr_map = kcalloc(adap->sge.egr_sz, 5026 sizeof(*adap->sge.egr_map), GFP_KERNEL); 5027 if (!adap->sge.egr_map) { 5028 ret = -ENOMEM; 5029 goto bye; 5030 } 5031 5032 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, 5033 sizeof(*adap->sge.ingr_map), GFP_KERNEL); 5034 if (!adap->sge.ingr_map) { 5035 ret = -ENOMEM; 5036 goto bye; 5037 } 5038 5039 /* Allocate the memory for the vaious egress queue bitmaps 5040 * ie starving_fl, txq_maperr and blocked_fl. 5041 */ 5042 adap->sge.starving_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); 5043 if (!adap->sge.starving_fl) { 5044 ret = -ENOMEM; 5045 goto bye; 5046 } 5047 5048 adap->sge.txq_maperr = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); 5049 if (!adap->sge.txq_maperr) { 5050 ret = -ENOMEM; 5051 goto bye; 5052 } 5053 5054#ifdef CONFIG_DEBUG_FS 5055 adap->sge.blocked_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL); 5056 if (!adap->sge.blocked_fl) { 5057 ret = -ENOMEM; 5058 goto bye; 5059 } 5060#endif 5061 5062 params[0] = FW_PARAM_PFVF(CLIP_START); 5063 params[1] = FW_PARAM_PFVF(CLIP_END); 5064 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 5065 if (ret < 0) 5066 goto bye; 5067 adap->clipt_start = val[0]; 5068 adap->clipt_end = val[1]; 5069 5070 /* Get the supported number of traffic classes */ 5071 params[0] = FW_PARAM_DEV(NUM_TM_CLASS); 5072 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 5073 if (ret < 0) { 5074 /* We couldn't retrieve the number of Traffic Classes 5075 * supported by the hardware/firmware. So we hard 5076 * code it here. 5077 */ 5078 adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16; 5079 } else { 5080 adap->params.nsched_cls = val[0]; 5081 } 5082 5083 /* query params related to active filter region */ 5084 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 5085 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 5086 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 5087 /* If Active filter size is set we enable establishing 5088 * offload connection through firmware work request 5089 */ 5090 if ((val[0] != val[1]) && (ret >= 0)) { 5091 adap->flags |= CXGB4_FW_OFLD_CONN; 5092 adap->tids.aftid_base = val[0]; 5093 adap->tids.aftid_end = val[1]; 5094 } 5095 5096 /* If we're running on newer firmware, let it know that we're 5097 * prepared to deal with encapsulated CPL messages. Older 5098 * firmware won't understand this and we'll just get 5099 * unencapsulated messages ... 5100 */ 5101 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 5102 val[0] = 1; 5103 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 5104 5105 /* 5106 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 5107 * capability. Earlier versions of the firmware didn't have the 5108 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 5109 * permission to use ULPTX MEMWRITE DSGL. 5110 */ 5111 if (is_t4(adap->params.chip)) { 5112 adap->params.ulptx_memwrite_dsgl = false; 5113 } else { 5114 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 5115 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5116 1, params, val); 5117 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 5118 } 5119 5120 /* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */ 5121 params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); 5122 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5123 1, params, val); 5124 adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0); 5125 5126 /* See if FW supports FW_FILTER2 work request */ 5127 if (is_t4(adap->params.chip)) { 5128 adap->params.filter2_wr_support = false; 5129 } else { 5130 params[0] = FW_PARAM_DEV(FILTER2_WR); 5131 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5132 1, params, val); 5133 adap->params.filter2_wr_support = (ret == 0 && val[0] != 0); 5134 } 5135 5136 /* Check if FW supports returning vin and smt index. 5137 * If this is not supported, driver will interpret 5138 * these values from viid. 5139 */ 5140 params[0] = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN); 5141 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5142 1, params, val); 5143 adap->params.viid_smt_extn_support = (ret == 0 && val[0] != 0); 5144 5145 /* 5146 * Get device capabilities so we can determine what resources we need 5147 * to manage. 5148 */ 5149 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5150 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5151 FW_CMD_REQUEST_F | FW_CMD_READ_F); 5152 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5153 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 5154 &caps_cmd); 5155 if (ret < 0) 5156 goto bye; 5157 5158 /* hash filter has some mandatory register settings to be tested and for 5159 * that it needs to test whether offload is enabled or not, hence 5160 * checking and setting it here. 5161 */ 5162 if (caps_cmd.ofldcaps) 5163 adap->params.offload = 1; 5164 5165 if (caps_cmd.ofldcaps || 5166 (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) || 5167 (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD))) { 5168 /* query offload-related parameters */ 5169 params[0] = FW_PARAM_DEV(NTID); 5170 params[1] = FW_PARAM_PFVF(SERVER_START); 5171 params[2] = FW_PARAM_PFVF(SERVER_END); 5172 params[3] = FW_PARAM_PFVF(TDDP_START); 5173 params[4] = FW_PARAM_PFVF(TDDP_END); 5174 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 5175 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 5176 params, val); 5177 if (ret < 0) 5178 goto bye; 5179 adap->tids.ntids = val[0]; 5180 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 5181 adap->tids.stid_base = val[1]; 5182 adap->tids.nstids = val[2] - val[1] + 1; 5183 /* 5184 * Setup server filter region. Divide the available filter 5185 * region into two parts. Regular filters get 1/3rd and server 5186 * filters get 2/3rd part. This is only enabled if workarond 5187 * path is enabled. 5188 * 1. For regular filters. 5189 * 2. Server filter: This are special filters which are used 5190 * to redirect SYN packets to offload queue. 5191 */ 5192 if (adap->flags & CXGB4_FW_OFLD_CONN && !is_bypass(adap)) { 5193 adap->tids.sftid_base = adap->tids.ftid_base + 5194 DIV_ROUND_UP(adap->tids.nftids, 3); 5195 adap->tids.nsftids = adap->tids.nftids - 5196 DIV_ROUND_UP(adap->tids.nftids, 3); 5197 adap->tids.nftids = adap->tids.sftid_base - 5198 adap->tids.ftid_base; 5199 } 5200 adap->vres.ddp.start = val[3]; 5201 adap->vres.ddp.size = val[4] - val[3] + 1; 5202 adap->params.ofldq_wr_cred = val[5]; 5203 5204 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) { 5205 init_hash_filter(adap); 5206 } else { 5207 adap->num_ofld_uld += 1; 5208 } 5209 5210 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD)) { 5211 params[0] = FW_PARAM_PFVF(ETHOFLD_START); 5212 params[1] = FW_PARAM_PFVF(ETHOFLD_END); 5213 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 5214 params, val); 5215 if (!ret) { 5216 adap->tids.eotid_base = val[0]; 5217 adap->tids.neotids = min_t(u32, MAX_ATIDS, 5218 val[1] - val[0] + 1); 5219 adap->params.ethofld = 1; 5220 } 5221 } 5222 } 5223 if (caps_cmd.rdmacaps) { 5224 params[0] = FW_PARAM_PFVF(STAG_START); 5225 params[1] = FW_PARAM_PFVF(STAG_END); 5226 params[2] = FW_PARAM_PFVF(RQ_START); 5227 params[3] = FW_PARAM_PFVF(RQ_END); 5228 params[4] = FW_PARAM_PFVF(PBL_START); 5229 params[5] = FW_PARAM_PFVF(PBL_END); 5230 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 5231 params, val); 5232 if (ret < 0) 5233 goto bye; 5234 adap->vres.stag.start = val[0]; 5235 adap->vres.stag.size = val[1] - val[0] + 1; 5236 adap->vres.rq.start = val[2]; 5237 adap->vres.rq.size = val[3] - val[2] + 1; 5238 adap->vres.pbl.start = val[4]; 5239 adap->vres.pbl.size = val[5] - val[4] + 1; 5240 5241 params[0] = FW_PARAM_PFVF(SRQ_START); 5242 params[1] = FW_PARAM_PFVF(SRQ_END); 5243 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 5244 params, val); 5245 if (!ret) { 5246 adap->vres.srq.start = val[0]; 5247 adap->vres.srq.size = val[1] - val[0] + 1; 5248 } 5249 if (adap->vres.srq.size) { 5250 adap->srq = t4_init_srq(adap->vres.srq.size); 5251 if (!adap->srq) 5252 dev_warn(&adap->pdev->dev, "could not allocate SRQ, continuing\n"); 5253 } 5254 5255 params[0] = FW_PARAM_PFVF(SQRQ_START); 5256 params[1] = FW_PARAM_PFVF(SQRQ_END); 5257 params[2] = FW_PARAM_PFVF(CQ_START); 5258 params[3] = FW_PARAM_PFVF(CQ_END); 5259 params[4] = FW_PARAM_PFVF(OCQ_START); 5260 params[5] = FW_PARAM_PFVF(OCQ_END); 5261 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, 5262 val); 5263 if (ret < 0) 5264 goto bye; 5265 adap->vres.qp.start = val[0]; 5266 adap->vres.qp.size = val[1] - val[0] + 1; 5267 adap->vres.cq.start = val[2]; 5268 adap->vres.cq.size = val[3] - val[2] + 1; 5269 adap->vres.ocq.start = val[4]; 5270 adap->vres.ocq.size = val[5] - val[4] + 1; 5271 5272 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 5273 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 5274 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, 5275 val); 5276 if (ret < 0) { 5277 adap->params.max_ordird_qp = 8; 5278 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 5279 ret = 0; 5280 } else { 5281 adap->params.max_ordird_qp = val[0]; 5282 adap->params.max_ird_adapter = val[1]; 5283 } 5284 dev_info(adap->pdev_dev, 5285 "max_ordird_qp %d max_ird_adapter %d\n", 5286 adap->params.max_ordird_qp, 5287 adap->params.max_ird_adapter); 5288 5289 /* Enable write_with_immediate if FW supports it */ 5290 params[0] = FW_PARAM_DEV(RDMA_WRITE_WITH_IMM); 5291 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, 5292 val); 5293 adap->params.write_w_imm_support = (ret == 0 && val[0] != 0); 5294 5295 /* Enable write_cmpl if FW supports it */ 5296 params[0] = FW_PARAM_DEV(RI_WRITE_CMPL_WR); 5297 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, 5298 val); 5299 adap->params.write_cmpl_support = (ret == 0 && val[0] != 0); 5300 adap->num_ofld_uld += 2; 5301 } 5302 if (caps_cmd.iscsicaps) { 5303 params[0] = FW_PARAM_PFVF(ISCSI_START); 5304 params[1] = FW_PARAM_PFVF(ISCSI_END); 5305 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 5306 params, val); 5307 if (ret < 0) 5308 goto bye; 5309 adap->vres.iscsi.start = val[0]; 5310 adap->vres.iscsi.size = val[1] - val[0] + 1; 5311 if (is_t6(adap->params.chip)) { 5312 params[0] = FW_PARAM_PFVF(PPOD_EDRAM_START); 5313 params[1] = FW_PARAM_PFVF(PPOD_EDRAM_END); 5314 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 5315 params, val); 5316 if (!ret) { 5317 adap->vres.ppod_edram.start = val[0]; 5318 adap->vres.ppod_edram.size = 5319 val[1] - val[0] + 1; 5320 5321 dev_info(adap->pdev_dev, 5322 "ppod edram start 0x%x end 0x%x size 0x%x\n", 5323 val[0], val[1], 5324 adap->vres.ppod_edram.size); 5325 } 5326 } 5327 /* LIO target and cxgb4i initiaitor */ 5328 adap->num_ofld_uld += 2; 5329 } 5330 if (caps_cmd.cryptocaps) { 5331 if (ntohs(caps_cmd.cryptocaps) & 5332 FW_CAPS_CONFIG_CRYPTO_LOOKASIDE) { 5333 params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); 5334 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5335 2, params, val); 5336 if (ret < 0) { 5337 if (ret != -EINVAL) 5338 goto bye; 5339 } else { 5340 adap->vres.ncrypto_fc = val[0]; 5341 } 5342 adap->num_ofld_uld += 1; 5343 } 5344 if (ntohs(caps_cmd.cryptocaps) & 5345 FW_CAPS_CONFIG_TLS_INLINE) { 5346 params[0] = FW_PARAM_PFVF(TLS_START); 5347 params[1] = FW_PARAM_PFVF(TLS_END); 5348 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 5349 2, params, val); 5350 if (ret < 0) 5351 goto bye; 5352 adap->vres.key.start = val[0]; 5353 adap->vres.key.size = val[1] - val[0] + 1; 5354 adap->num_uld += 1; 5355 } 5356 adap->params.crypto = ntohs(caps_cmd.cryptocaps); 5357 } 5358 5359 /* The MTU/MSS Table is initialized by now, so load their values. If 5360 * we're initializing the adapter, then we'll make any modifications 5361 * we want to the MTU/MSS Table and also initialize the congestion 5362 * parameters. 5363 */ 5364 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 5365 if (state != DEV_STATE_INIT) { 5366 int i; 5367 5368 /* The default MTU Table contains values 1492 and 1500. 5369 * However, for TCP, it's better to have two values which are 5370 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 5371 * This allows us to have a TCP Data Payload which is a 5372 * multiple of 8 regardless of what combination of TCP Options 5373 * are in use (always a multiple of 4 bytes) which is 5374 * important for performance reasons. For instance, if no 5375 * options are in use, then we have a 20-byte IP header and a 5376 * 20-byte TCP header. In this case, a 1500-byte MSS would 5377 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 5378 * which is not a multiple of 8. So using an MSS of 1488 in 5379 * this case results in a TCP Data Payload of 1448 bytes which 5380 * is a multiple of 8. On the other hand, if 12-byte TCP Time 5381 * Stamps have been negotiated, then an MTU of 1500 bytes 5382 * results in a TCP Data Payload of 1448 bytes which, as 5383 * above, is a multiple of 8 bytes ... 5384 */ 5385 for (i = 0; i < NMTUS; i++) 5386 if (adap->params.mtus[i] == 1492) { 5387 adap->params.mtus[i] = 1488; 5388 break; 5389 } 5390 5391 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 5392 adap->params.b_wnd); 5393 } 5394 t4_init_sge_params(adap); 5395 adap->flags |= CXGB4_FW_OK; 5396 t4_init_tp_params(adap, true); 5397 return 0; 5398 5399 /* 5400 * Something bad happened. If a command timed out or failed with EIO 5401 * FW does not operate within its spec or something catastrophic 5402 * happened to HW/FW, stop issuing commands. 5403 */ 5404bye: 5405 adap_free_hma_mem(adap); 5406 kfree(adap->sge.egr_map); 5407 kfree(adap->sge.ingr_map); 5408 bitmap_free(adap->sge.starving_fl); 5409 bitmap_free(adap->sge.txq_maperr); 5410#ifdef CONFIG_DEBUG_FS 5411 bitmap_free(adap->sge.blocked_fl); 5412#endif 5413 if (ret != -ETIMEDOUT && ret != -EIO) 5414 t4_fw_bye(adap, adap->mbox); 5415 return ret; 5416} 5417 5418/* EEH callbacks */ 5419 5420static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 5421 pci_channel_state_t state) 5422{ 5423 int i; 5424 struct adapter *adap = pci_get_drvdata(pdev); 5425 5426 if (!adap) 5427 goto out; 5428 5429 rtnl_lock(); 5430 adap->flags &= ~CXGB4_FW_OK; 5431 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 5432 spin_lock(&adap->stats_lock); 5433 for_each_port(adap, i) { 5434 struct net_device *dev = adap->port[i]; 5435 if (dev) { 5436 netif_device_detach(dev); 5437 netif_carrier_off(dev); 5438 } 5439 } 5440 spin_unlock(&adap->stats_lock); 5441 disable_interrupts(adap); 5442 if (adap->flags & CXGB4_FULL_INIT_DONE) 5443 cxgb_down(adap); 5444 rtnl_unlock(); 5445 if ((adap->flags & CXGB4_DEV_ENABLED)) { 5446 pci_disable_device(pdev); 5447 adap->flags &= ~CXGB4_DEV_ENABLED; 5448 } 5449out: return state == pci_channel_io_perm_failure ? 5450 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 5451} 5452 5453static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 5454{ 5455 int i, ret; 5456 struct fw_caps_config_cmd c; 5457 struct adapter *adap = pci_get_drvdata(pdev); 5458 5459 if (!adap) { 5460 pci_restore_state(pdev); 5461 return PCI_ERS_RESULT_RECOVERED; 5462 } 5463 5464 if (!(adap->flags & CXGB4_DEV_ENABLED)) { 5465 if (pci_enable_device(pdev)) { 5466 dev_err(&pdev->dev, "Cannot reenable PCI " 5467 "device after reset\n"); 5468 return PCI_ERS_RESULT_DISCONNECT; 5469 } 5470 adap->flags |= CXGB4_DEV_ENABLED; 5471 } 5472 5473 pci_set_master(pdev); 5474 pci_restore_state(pdev); 5475 5476 if (t4_wait_dev_ready(adap->regs) < 0) 5477 return PCI_ERS_RESULT_DISCONNECT; 5478 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) 5479 return PCI_ERS_RESULT_DISCONNECT; 5480 adap->flags |= CXGB4_FW_OK; 5481 if (adap_init1(adap, &c)) 5482 return PCI_ERS_RESULT_DISCONNECT; 5483 5484 for_each_port(adap, i) { 5485 struct port_info *pi = adap2pinfo(adap, i); 5486 u8 vivld = 0, vin = 0; 5487 5488 ret = t4_alloc_vi(adap, adap->mbox, pi->tx_chan, adap->pf, 0, 1, 5489 NULL, NULL, &vivld, &vin); 5490 if (ret < 0) 5491 return PCI_ERS_RESULT_DISCONNECT; 5492 pi->viid = ret; 5493 pi->xact_addr_filt = -1; 5494 /* If fw supports returning the VIN as part of FW_VI_CMD, 5495 * save the returned values. 5496 */ 5497 if (adap->params.viid_smt_extn_support) { 5498 pi->vivld = vivld; 5499 pi->vin = vin; 5500 } else { 5501 /* Retrieve the values from VIID */ 5502 pi->vivld = FW_VIID_VIVLD_G(pi->viid); 5503 pi->vin = FW_VIID_VIN_G(pi->viid); 5504 } 5505 } 5506 5507 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 5508 adap->params.b_wnd); 5509 setup_memwin(adap); 5510 if (cxgb_up(adap)) 5511 return PCI_ERS_RESULT_DISCONNECT; 5512 return PCI_ERS_RESULT_RECOVERED; 5513} 5514 5515static void eeh_resume(struct pci_dev *pdev) 5516{ 5517 int i; 5518 struct adapter *adap = pci_get_drvdata(pdev); 5519 5520 if (!adap) 5521 return; 5522 5523 rtnl_lock(); 5524 for_each_port(adap, i) { 5525 struct net_device *dev = adap->port[i]; 5526 if (dev) { 5527 if (netif_running(dev)) { 5528 link_start(dev); 5529 cxgb_set_rxmode(dev); 5530 } 5531 netif_device_attach(dev); 5532 } 5533 } 5534 rtnl_unlock(); 5535} 5536 5537static void eeh_reset_prepare(struct pci_dev *pdev) 5538{ 5539 struct adapter *adapter = pci_get_drvdata(pdev); 5540 int i; 5541 5542 if (adapter->pf != 4) 5543 return; 5544 5545 adapter->flags &= ~CXGB4_FW_OK; 5546 5547 notify_ulds(adapter, CXGB4_STATE_DOWN); 5548 5549 for_each_port(adapter, i) 5550 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5551 cxgb_close(adapter->port[i]); 5552 5553 disable_interrupts(adapter); 5554 cxgb4_free_mps_ref_entries(adapter); 5555 5556 adap_free_hma_mem(adapter); 5557 5558 if (adapter->flags & CXGB4_FULL_INIT_DONE) 5559 cxgb_down(adapter); 5560} 5561 5562static void eeh_reset_done(struct pci_dev *pdev) 5563{ 5564 struct adapter *adapter = pci_get_drvdata(pdev); 5565 int err, i; 5566 5567 if (adapter->pf != 4) 5568 return; 5569 5570 err = t4_wait_dev_ready(adapter->regs); 5571 if (err < 0) { 5572 dev_err(adapter->pdev_dev, 5573 "Device not ready, err %d", err); 5574 return; 5575 } 5576 5577 setup_memwin(adapter); 5578 5579 err = adap_init0(adapter, 1); 5580 if (err) { 5581 dev_err(adapter->pdev_dev, 5582 "Adapter init failed, err %d", err); 5583 return; 5584 } 5585 5586 setup_memwin_rdma(adapter); 5587 5588 if (adapter->flags & CXGB4_FW_OK) { 5589 err = t4_port_init(adapter, adapter->pf, adapter->pf, 0); 5590 if (err) { 5591 dev_err(adapter->pdev_dev, 5592 "Port init failed, err %d", err); 5593 return; 5594 } 5595 } 5596 5597 err = cfg_queues(adapter); 5598 if (err) { 5599 dev_err(adapter->pdev_dev, 5600 "Config queues failed, err %d", err); 5601 return; 5602 } 5603 5604 cxgb4_init_mps_ref_entries(adapter); 5605 5606 err = setup_fw_sge_queues(adapter); 5607 if (err) { 5608 dev_err(adapter->pdev_dev, 5609 "FW sge queue allocation failed, err %d", err); 5610 return; 5611 } 5612 5613 for_each_port(adapter, i) 5614 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5615 cxgb_open(adapter->port[i]); 5616} 5617 5618static const struct pci_error_handlers cxgb4_eeh = { 5619 .error_detected = eeh_err_detected, 5620 .slot_reset = eeh_slot_reset, 5621 .resume = eeh_resume, 5622 .reset_prepare = eeh_reset_prepare, 5623 .reset_done = eeh_reset_done, 5624}; 5625 5626/* Return true if the Link Configuration supports "High Speeds" (those greater 5627 * than 1Gb/s). 5628 */ 5629static inline bool is_x_10g_port(const struct link_config *lc) 5630{ 5631 unsigned int speeds, high_speeds; 5632 5633 speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); 5634 high_speeds = speeds & 5635 ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); 5636 5637 return high_speeds != 0; 5638} 5639 5640/* Perform default configuration of DMA queues depending on the number and type 5641 * of ports we found and the number of available CPUs. Most settings can be 5642 * modified by the admin prior to actual use. 5643 */ 5644static int cfg_queues(struct adapter *adap) 5645{ 5646 u32 avail_qsets, avail_eth_qsets, avail_uld_qsets; 5647 u32 ncpus = num_online_cpus(); 5648 u32 niqflint, neq, num_ulds; 5649 struct sge *s = &adap->sge; 5650 u32 i, n10g = 0, qidx = 0; 5651 u32 q10g = 0, q1g; 5652 5653 /* Reduce memory usage in kdump environment, disable all offload. */ 5654 if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) { 5655 adap->params.offload = 0; 5656 adap->params.crypto = 0; 5657 adap->params.ethofld = 0; 5658 } 5659 5660 /* Calculate the number of Ethernet Queue Sets available based on 5661 * resources provisioned for us. We always have an Asynchronous 5662 * Firmware Event Ingress Queue. If we're operating in MSI or Legacy 5663 * IRQ Pin Interrupt mode, then we'll also have a Forwarded Interrupt 5664 * Ingress Queue. Meanwhile, we need two Egress Queues for each 5665 * Queue Set: one for the Free List and one for the Ethernet TX Queue. 5666 * 5667 * Note that we should also take into account all of the various 5668 * Offload Queues. But, in any situation where we're operating in 5669 * a Resource Constrained Provisioning environment, doing any Offload 5670 * at all is problematic ... 5671 */ 5672 niqflint = adap->params.pfres.niqflint - 1; 5673 if (!(adap->flags & CXGB4_USING_MSIX)) 5674 niqflint--; 5675 neq = adap->params.pfres.neq / 2; 5676 avail_qsets = min(niqflint, neq); 5677 5678 if (avail_qsets < adap->params.nports) { 5679 dev_err(adap->pdev_dev, "avail_eth_qsets=%d < nports=%d\n", 5680 avail_qsets, adap->params.nports); 5681 return -ENOMEM; 5682 } 5683 5684 /* Count the number of 10Gb/s or better ports */ 5685 for_each_port(adap, i) 5686 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 5687 5688 avail_eth_qsets = min_t(u32, avail_qsets, MAX_ETH_QSETS); 5689 5690 /* We default to 1 queue per non-10G port and up to # of cores queues 5691 * per 10G port. 5692 */ 5693 if (n10g) 5694 q10g = (avail_eth_qsets - (adap->params.nports - n10g)) / n10g; 5695 5696#ifdef CONFIG_CHELSIO_T4_DCB 5697 /* For Data Center Bridging support we need to be able to support up 5698 * to 8 Traffic Priorities; each of which will be assigned to its 5699 * own TX Queue in order to prevent Head-Of-Line Blocking. 5700 */ 5701 q1g = 8; 5702 if (adap->params.nports * 8 > avail_eth_qsets) { 5703 dev_err(adap->pdev_dev, "DCB avail_eth_qsets=%d < %d!\n", 5704 avail_eth_qsets, adap->params.nports * 8); 5705 return -ENOMEM; 5706 } 5707 5708 if (adap->params.nports * ncpus < avail_eth_qsets) 5709 q10g = max(8U, ncpus); 5710 else 5711 q10g = max(8U, q10g); 5712 5713 while ((q10g * n10g) > 5714 (avail_eth_qsets - (adap->params.nports - n10g) * q1g)) 5715 q10g--; 5716 5717#else /* !CONFIG_CHELSIO_T4_DCB */ 5718 q1g = 1; 5719 q10g = min(q10g, ncpus); 5720#endif /* !CONFIG_CHELSIO_T4_DCB */ 5721 if (is_kdump_kernel()) { 5722 q10g = 1; 5723 q1g = 1; 5724 } 5725 5726 for_each_port(adap, i) { 5727 struct port_info *pi = adap2pinfo(adap, i); 5728 5729 pi->first_qset = qidx; 5730 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : q1g; 5731 qidx += pi->nqsets; 5732 } 5733 5734 s->ethqsets = qidx; 5735 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 5736 avail_qsets -= qidx; 5737 5738 if (is_uld(adap)) { 5739 /* For offload we use 1 queue/channel if all ports are up to 1G, 5740 * otherwise we divide all available queues amongst the channels 5741 * capped by the number of available cores. 5742 */ 5743 num_ulds = adap->num_uld + adap->num_ofld_uld; 5744 i = min_t(u32, MAX_OFLD_QSETS, ncpus); 5745 avail_uld_qsets = roundup(i, adap->params.nports); 5746 if (avail_qsets < num_ulds * adap->params.nports) { 5747 adap->params.offload = 0; 5748 adap->params.crypto = 0; 5749 s->ofldqsets = 0; 5750 } else if (avail_qsets < num_ulds * avail_uld_qsets || !n10g) { 5751 s->ofldqsets = adap->params.nports; 5752 } else { 5753 s->ofldqsets = avail_uld_qsets; 5754 } 5755 5756 avail_qsets -= num_ulds * s->ofldqsets; 5757 } 5758 5759 /* ETHOFLD Queues used for QoS offload should follow same 5760 * allocation scheme as normal Ethernet Queues. 5761 */ 5762 if (is_ethofld(adap)) { 5763 if (avail_qsets < s->max_ethqsets) { 5764 adap->params.ethofld = 0; 5765 s->eoqsets = 0; 5766 } else { 5767 s->eoqsets = s->max_ethqsets; 5768 } 5769 avail_qsets -= s->eoqsets; 5770 } 5771 5772 /* Mirror queues must follow same scheme as normal Ethernet 5773 * Queues, when there are enough queues available. Otherwise, 5774 * allocate at least 1 queue per port. If even 1 queue is not 5775 * available, then disable mirror queues support. 5776 */ 5777 if (avail_qsets >= s->max_ethqsets) 5778 s->mirrorqsets = s->max_ethqsets; 5779 else if (avail_qsets >= adap->params.nports) 5780 s->mirrorqsets = adap->params.nports; 5781 else 5782 s->mirrorqsets = 0; 5783 avail_qsets -= s->mirrorqsets; 5784 5785 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 5786 struct sge_eth_rxq *r = &s->ethrxq[i]; 5787 5788 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 5789 r->fl.size = 72; 5790 } 5791 5792 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 5793 s->ethtxq[i].q.size = 1024; 5794 5795 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 5796 s->ctrlq[i].q.size = 512; 5797 5798 if (!is_t4(adap->params.chip)) 5799 s->ptptxq.q.size = 8; 5800 5801 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 5802 init_rspq(adap, &s->intrq, 0, 1, 512, 64); 5803 5804 return 0; 5805} 5806 5807/* 5808 * Reduce the number of Ethernet queues across all ports to at most n. 5809 * n provides at least one queue per port. 5810 */ 5811static void reduce_ethqs(struct adapter *adap, int n) 5812{ 5813 int i; 5814 struct port_info *pi; 5815 5816 while (n < adap->sge.ethqsets) 5817 for_each_port(adap, i) { 5818 pi = adap2pinfo(adap, i); 5819 if (pi->nqsets > 1) { 5820 pi->nqsets--; 5821 adap->sge.ethqsets--; 5822 if (adap->sge.ethqsets <= n) 5823 break; 5824 } 5825 } 5826 5827 n = 0; 5828 for_each_port(adap, i) { 5829 pi = adap2pinfo(adap, i); 5830 pi->first_qset = n; 5831 n += pi->nqsets; 5832 } 5833} 5834 5835static int alloc_msix_info(struct adapter *adap, u32 num_vec) 5836{ 5837 struct msix_info *msix_info; 5838 5839 msix_info = kcalloc(num_vec, sizeof(*msix_info), GFP_KERNEL); 5840 if (!msix_info) 5841 return -ENOMEM; 5842 5843 adap->msix_bmap.msix_bmap = bitmap_zalloc(num_vec, GFP_KERNEL); 5844 if (!adap->msix_bmap.msix_bmap) { 5845 kfree(msix_info); 5846 return -ENOMEM; 5847 } 5848 5849 spin_lock_init(&adap->msix_bmap.lock); 5850 adap->msix_bmap.mapsize = num_vec; 5851 5852 adap->msix_info = msix_info; 5853 return 0; 5854} 5855 5856static void free_msix_info(struct adapter *adap) 5857{ 5858 bitmap_free(adap->msix_bmap.msix_bmap); 5859 kfree(adap->msix_info); 5860} 5861 5862int cxgb4_get_msix_idx_from_bmap(struct adapter *adap) 5863{ 5864 struct msix_bmap *bmap = &adap->msix_bmap; 5865 unsigned int msix_idx; 5866 unsigned long flags; 5867 5868 spin_lock_irqsave(&bmap->lock, flags); 5869 msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize); 5870 if (msix_idx < bmap->mapsize) { 5871 __set_bit(msix_idx, bmap->msix_bmap); 5872 } else { 5873 spin_unlock_irqrestore(&bmap->lock, flags); 5874 return -ENOSPC; 5875 } 5876 5877 spin_unlock_irqrestore(&bmap->lock, flags); 5878 return msix_idx; 5879} 5880 5881void cxgb4_free_msix_idx_in_bmap(struct adapter *adap, 5882 unsigned int msix_idx) 5883{ 5884 struct msix_bmap *bmap = &adap->msix_bmap; 5885 unsigned long flags; 5886 5887 spin_lock_irqsave(&bmap->lock, flags); 5888 __clear_bit(msix_idx, bmap->msix_bmap); 5889 spin_unlock_irqrestore(&bmap->lock, flags); 5890} 5891 5892/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 5893#define EXTRA_VECS 2 5894 5895static int enable_msix(struct adapter *adap) 5896{ 5897 u32 eth_need, uld_need = 0, ethofld_need = 0, mirror_need = 0; 5898 u32 ethqsets = 0, ofldqsets = 0, eoqsets = 0, mirrorqsets = 0; 5899 u8 num_uld = 0, nchan = adap->params.nports; 5900 u32 i, want, need, num_vec; 5901 struct sge *s = &adap->sge; 5902 struct msix_entry *entries; 5903 struct port_info *pi; 5904 int allocated, ret; 5905 5906 want = s->max_ethqsets; 5907#ifdef CONFIG_CHELSIO_T4_DCB 5908 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 5909 * each port. 5910 */ 5911 need = 8 * nchan; 5912#else 5913 need = nchan; 5914#endif 5915 eth_need = need; 5916 if (is_uld(adap)) { 5917 num_uld = adap->num_ofld_uld + adap->num_uld; 5918 want += num_uld * s->ofldqsets; 5919 uld_need = num_uld * nchan; 5920 need += uld_need; 5921 } 5922 5923 if (is_ethofld(adap)) { 5924 want += s->eoqsets; 5925 ethofld_need = eth_need; 5926 need += ethofld_need; 5927 } 5928 5929 if (s->mirrorqsets) { 5930 want += s->mirrorqsets; 5931 mirror_need = nchan; 5932 need += mirror_need; 5933 } 5934 5935 want += EXTRA_VECS; 5936 need += EXTRA_VECS; 5937 5938 entries = kmalloc_array(want, sizeof(*entries), GFP_KERNEL); 5939 if (!entries) 5940 return -ENOMEM; 5941 5942 for (i = 0; i < want; i++) 5943 entries[i].entry = i; 5944 5945 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 5946 if (allocated < 0) { 5947 /* Disable offload and attempt to get vectors for NIC 5948 * only mode. 5949 */ 5950 want = s->max_ethqsets + EXTRA_VECS; 5951 need = eth_need + EXTRA_VECS; 5952 allocated = pci_enable_msix_range(adap->pdev, entries, 5953 need, want); 5954 if (allocated < 0) { 5955 dev_info(adap->pdev_dev, 5956 "Disabling MSI-X due to insufficient MSI-X vectors\n"); 5957 ret = allocated; 5958 goto out_free; 5959 } 5960 5961 dev_info(adap->pdev_dev, 5962 "Disabling offload due to insufficient MSI-X vectors\n"); 5963 adap->params.offload = 0; 5964 adap->params.crypto = 0; 5965 adap->params.ethofld = 0; 5966 s->ofldqsets = 0; 5967 s->eoqsets = 0; 5968 s->mirrorqsets = 0; 5969 uld_need = 0; 5970 ethofld_need = 0; 5971 mirror_need = 0; 5972 } 5973 5974 num_vec = allocated; 5975 if (num_vec < want) { 5976 /* Distribute available vectors to the various queue groups. 5977 * Every group gets its minimum requirement and NIC gets top 5978 * priority for leftovers. 5979 */ 5980 ethqsets = eth_need; 5981 if (is_uld(adap)) 5982 ofldqsets = nchan; 5983 if (is_ethofld(adap)) 5984 eoqsets = ethofld_need; 5985 if (s->mirrorqsets) 5986 mirrorqsets = mirror_need; 5987 5988 num_vec -= need; 5989 while (num_vec) { 5990 if (num_vec < eth_need + ethofld_need || 5991 ethqsets > s->max_ethqsets) 5992 break; 5993 5994 for_each_port(adap, i) { 5995 pi = adap2pinfo(adap, i); 5996 if (pi->nqsets < 2) 5997 continue; 5998 5999 ethqsets++; 6000 num_vec--; 6001 if (ethofld_need) { 6002 eoqsets++; 6003 num_vec--; 6004 } 6005 } 6006 } 6007 6008 if (is_uld(adap)) { 6009 while (num_vec) { 6010 if (num_vec < uld_need || 6011 ofldqsets > s->ofldqsets) 6012 break; 6013 6014 ofldqsets++; 6015 num_vec -= uld_need; 6016 } 6017 } 6018 6019 if (s->mirrorqsets) { 6020 while (num_vec) { 6021 if (num_vec < mirror_need || 6022 mirrorqsets > s->mirrorqsets) 6023 break; 6024 6025 mirrorqsets++; 6026 num_vec -= mirror_need; 6027 } 6028 } 6029 } else { 6030 ethqsets = s->max_ethqsets; 6031 if (is_uld(adap)) 6032 ofldqsets = s->ofldqsets; 6033 if (is_ethofld(adap)) 6034 eoqsets = s->eoqsets; 6035 if (s->mirrorqsets) 6036 mirrorqsets = s->mirrorqsets; 6037 } 6038 6039 if (ethqsets < s->max_ethqsets) { 6040 s->max_ethqsets = ethqsets; 6041 reduce_ethqs(adap, ethqsets); 6042 } 6043 6044 if (is_uld(adap)) { 6045 s->ofldqsets = ofldqsets; 6046 s->nqs_per_uld = s->ofldqsets; 6047 } 6048 6049 if (is_ethofld(adap)) 6050 s->eoqsets = eoqsets; 6051 6052 if (s->mirrorqsets) { 6053 s->mirrorqsets = mirrorqsets; 6054 for_each_port(adap, i) { 6055 pi = adap2pinfo(adap, i); 6056 pi->nmirrorqsets = s->mirrorqsets / nchan; 6057 mutex_init(&pi->vi_mirror_mutex); 6058 } 6059 } 6060 6061 /* map for msix */ 6062 ret = alloc_msix_info(adap, allocated); 6063 if (ret) 6064 goto out_disable_msix; 6065 6066 for (i = 0; i < allocated; i++) { 6067 adap->msix_info[i].vec = entries[i].vector; 6068 adap->msix_info[i].idx = i; 6069 } 6070 6071 dev_info(adap->pdev_dev, 6072 "%d MSI-X vectors allocated, nic %d eoqsets %d per uld %d mirrorqsets %d\n", 6073 allocated, s->max_ethqsets, s->eoqsets, s->nqs_per_uld, 6074 s->mirrorqsets); 6075 6076 kfree(entries); 6077 return 0; 6078 6079out_disable_msix: 6080 pci_disable_msix(adap->pdev); 6081 6082out_free: 6083 kfree(entries); 6084 return ret; 6085} 6086 6087#undef EXTRA_VECS 6088 6089static int init_rss(struct adapter *adap) 6090{ 6091 unsigned int i; 6092 int err; 6093 6094 err = t4_init_rss_mode(adap, adap->mbox); 6095 if (err) 6096 return err; 6097 6098 for_each_port(adap, i) { 6099 struct port_info *pi = adap2pinfo(adap, i); 6100 6101 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 6102 if (!pi->rss) 6103 return -ENOMEM; 6104 } 6105 return 0; 6106} 6107 6108/* Dump basic information about the adapter */ 6109static void print_adapter_info(struct adapter *adapter) 6110{ 6111 /* Hardware/Firmware/etc. Version/Revision IDs */ 6112 t4_dump_version_info(adapter); 6113 6114 /* Software/Hardware configuration */ 6115 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", 6116 is_offload(adapter) ? "R" : "", 6117 ((adapter->flags & CXGB4_USING_MSIX) ? "MSI-X" : 6118 (adapter->flags & CXGB4_USING_MSI) ? "MSI" : ""), 6119 is_offload(adapter) ? "Offload" : "non-Offload"); 6120} 6121 6122static void print_port_info(const struct net_device *dev) 6123{ 6124 char buf[80]; 6125 char *bufp = buf; 6126 const struct port_info *pi = netdev_priv(dev); 6127 const struct adapter *adap = pi->adapter; 6128 6129 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) 6130 bufp += sprintf(bufp, "100M/"); 6131 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) 6132 bufp += sprintf(bufp, "1G/"); 6133 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) 6134 bufp += sprintf(bufp, "10G/"); 6135 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) 6136 bufp += sprintf(bufp, "25G/"); 6137 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) 6138 bufp += sprintf(bufp, "40G/"); 6139 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) 6140 bufp += sprintf(bufp, "50G/"); 6141 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) 6142 bufp += sprintf(bufp, "100G/"); 6143 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G) 6144 bufp += sprintf(bufp, "200G/"); 6145 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G) 6146 bufp += sprintf(bufp, "400G/"); 6147 if (bufp != buf) 6148 --bufp; 6149 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 6150 6151 netdev_info(dev, "Chelsio %s %s\n", adap->params.vpd.id, buf); 6152} 6153 6154/* 6155 * Free the following resources: 6156 * - memory used for tables 6157 * - MSI/MSI-X 6158 * - net devices 6159 * - resources FW is holding for us 6160 */ 6161static void free_some_resources(struct adapter *adapter) 6162{ 6163 unsigned int i; 6164 6165 kvfree(adapter->smt); 6166 kvfree(adapter->l2t); 6167 kvfree(adapter->srq); 6168 t4_cleanup_sched(adapter); 6169 kvfree(adapter->tids.tid_tab); 6170 cxgb4_cleanup_tc_matchall(adapter); 6171 cxgb4_cleanup_tc_mqprio(adapter); 6172 cxgb4_cleanup_tc_flower(adapter); 6173 cxgb4_cleanup_tc_u32(adapter); 6174 cxgb4_cleanup_ethtool_filters(adapter); 6175 kfree(adapter->sge.egr_map); 6176 kfree(adapter->sge.ingr_map); 6177 bitmap_free(adapter->sge.starving_fl); 6178 bitmap_free(adapter->sge.txq_maperr); 6179#ifdef CONFIG_DEBUG_FS 6180 bitmap_free(adapter->sge.blocked_fl); 6181#endif 6182 disable_msi(adapter); 6183 6184 for_each_port(adapter, i) 6185 if (adapter->port[i]) { 6186 struct port_info *pi = adap2pinfo(adapter, i); 6187 6188 if (pi->viid != 0) 6189 t4_free_vi(adapter, adapter->mbox, adapter->pf, 6190 0, pi->viid); 6191 kfree(adap2pinfo(adapter, i)->rss); 6192 free_netdev(adapter->port[i]); 6193 } 6194 if (adapter->flags & CXGB4_FW_OK) 6195 t4_fw_bye(adapter, adapter->pf); 6196} 6197 6198#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | \ 6199 NETIF_F_GSO_UDP_L4) 6200#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 6201 NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 6202#define SEGMENT_SIZE 128 6203 6204static int t4_get_chip_type(struct adapter *adap, int ver) 6205{ 6206 u32 pl_rev = REV_G(t4_read_reg(adap, PL_REV_A)); 6207 6208 switch (ver) { 6209 case CHELSIO_T4: 6210 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); 6211 case CHELSIO_T5: 6212 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); 6213 case CHELSIO_T6: 6214 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); 6215 default: 6216 break; 6217 } 6218 return -EINVAL; 6219} 6220 6221#ifdef CONFIG_PCI_IOV 6222static void cxgb4_mgmt_setup(struct net_device *dev) 6223{ 6224 dev->type = ARPHRD_NONE; 6225 dev->mtu = 0; 6226 dev->hard_header_len = 0; 6227 dev->addr_len = 0; 6228 dev->tx_queue_len = 0; 6229 dev->flags |= IFF_NOARP; 6230 dev->priv_flags |= IFF_NO_QUEUE; 6231 6232 /* Initialize the device structure. */ 6233 dev->netdev_ops = &cxgb4_mgmt_netdev_ops; 6234 dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; 6235} 6236 6237static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) 6238{ 6239 struct adapter *adap = pci_get_drvdata(pdev); 6240 int err = 0; 6241 int current_vfs = pci_num_vf(pdev); 6242 u32 pcie_fw; 6243 6244 pcie_fw = readl(adap->regs + PCIE_FW_A); 6245 /* Check if fw is initialized */ 6246 if (!(pcie_fw & PCIE_FW_INIT_F)) { 6247 dev_warn(&pdev->dev, "Device not initialized\n"); 6248 return -EOPNOTSUPP; 6249 } 6250 6251 /* If any of the VF's is already assigned to Guest OS, then 6252 * SRIOV for the same cannot be modified 6253 */ 6254 if (current_vfs && pci_vfs_assigned(pdev)) { 6255 dev_err(&pdev->dev, 6256 "Cannot modify SR-IOV while VFs are assigned\n"); 6257 return current_vfs; 6258 } 6259 /* Note that the upper-level code ensures that we're never called with 6260 * a non-zero "num_vfs" when we already have VFs instantiated. But 6261 * it never hurts to code defensively. 6262 */ 6263 if (num_vfs != 0 && current_vfs != 0) 6264 return -EBUSY; 6265 6266 /* Nothing to do for no change. */ 6267 if (num_vfs == current_vfs) 6268 return num_vfs; 6269 6270 /* Disable SRIOV when zero is passed. */ 6271 if (!num_vfs) { 6272 pci_disable_sriov(pdev); 6273 /* free VF Management Interface */ 6274 unregister_netdev(adap->port[0]); 6275 free_netdev(adap->port[0]); 6276 adap->port[0] = NULL; 6277 6278 /* free VF resources */ 6279 adap->num_vfs = 0; 6280 kfree(adap->vfinfo); 6281 adap->vfinfo = NULL; 6282 return 0; 6283 } 6284 6285 if (!current_vfs) { 6286 struct fw_pfvf_cmd port_cmd, port_rpl; 6287 struct net_device *netdev; 6288 unsigned int pmask, port; 6289 struct pci_dev *pbridge; 6290 struct port_info *pi; 6291 char name[IFNAMSIZ]; 6292 u32 devcap2; 6293 u16 flags; 6294 6295 /* If we want to instantiate Virtual Functions, then our 6296 * parent bridge's PCI-E needs to support Alternative Routing 6297 * ID (ARI) because our VFs will show up at function offset 8 6298 * and above. 6299 */ 6300 pbridge = pdev->bus->self; 6301 pcie_capability_read_word(pbridge, PCI_EXP_FLAGS, &flags); 6302 pcie_capability_read_dword(pbridge, PCI_EXP_DEVCAP2, &devcap2); 6303 6304 if ((flags & PCI_EXP_FLAGS_VERS) < 2 || 6305 !(devcap2 & PCI_EXP_DEVCAP2_ARI)) { 6306 /* Our parent bridge does not support ARI so issue a 6307 * warning and skip instantiating the VFs. They 6308 * won't be reachable. 6309 */ 6310 dev_warn(&pdev->dev, "Parent bridge %02x:%02x.%x doesn't support ARI; can't instantiate Virtual Functions\n", 6311 pbridge->bus->number, PCI_SLOT(pbridge->devfn), 6312 PCI_FUNC(pbridge->devfn)); 6313 return -ENOTSUPP; 6314 } 6315 memset(&port_cmd, 0, sizeof(port_cmd)); 6316 port_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | 6317 FW_CMD_REQUEST_F | 6318 FW_CMD_READ_F | 6319 FW_PFVF_CMD_PFN_V(adap->pf) | 6320 FW_PFVF_CMD_VFN_V(0)); 6321 port_cmd.retval_len16 = cpu_to_be32(FW_LEN16(port_cmd)); 6322 err = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd), 6323 &port_rpl); 6324 if (err) 6325 return err; 6326 pmask = FW_PFVF_CMD_PMASK_G(be32_to_cpu(port_rpl.type_to_neq)); 6327 port = ffs(pmask) - 1; 6328 /* Allocate VF Management Interface. */ 6329 snprintf(name, IFNAMSIZ, "mgmtpf%d,%d", adap->adap_idx, 6330 adap->pf); 6331 netdev = alloc_netdev(sizeof(struct port_info), 6332 name, NET_NAME_UNKNOWN, cxgb4_mgmt_setup); 6333 if (!netdev) 6334 return -ENOMEM; 6335 6336 pi = netdev_priv(netdev); 6337 pi->adapter = adap; 6338 pi->lport = port; 6339 pi->tx_chan = port; 6340 SET_NETDEV_DEV(netdev, &pdev->dev); 6341 6342 adap->port[0] = netdev; 6343 pi->port_id = 0; 6344 6345 err = register_netdev(adap->port[0]); 6346 if (err) { 6347 pr_info("Unable to register VF mgmt netdev %s\n", name); 6348 free_netdev(adap->port[0]); 6349 adap->port[0] = NULL; 6350 return err; 6351 } 6352 /* Allocate and set up VF Information. */ 6353 adap->vfinfo = kcalloc(pci_sriov_get_totalvfs(pdev), 6354 sizeof(struct vf_info), GFP_KERNEL); 6355 if (!adap->vfinfo) { 6356 unregister_netdev(adap->port[0]); 6357 free_netdev(adap->port[0]); 6358 adap->port[0] = NULL; 6359 return -ENOMEM; 6360 } 6361 cxgb4_mgmt_fill_vf_station_mac_addr(adap); 6362 } 6363 /* Instantiate the requested number of VFs. */ 6364 err = pci_enable_sriov(pdev, num_vfs); 6365 if (err) { 6366 pr_info("Unable to instantiate %d VFs\n", num_vfs); 6367 if (!current_vfs) { 6368 unregister_netdev(adap->port[0]); 6369 free_netdev(adap->port[0]); 6370 adap->port[0] = NULL; 6371 kfree(adap->vfinfo); 6372 adap->vfinfo = NULL; 6373 } 6374 return err; 6375 } 6376 6377 adap->num_vfs = num_vfs; 6378 return num_vfs; 6379} 6380#endif /* CONFIG_PCI_IOV */ 6381 6382#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) || IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) 6383 6384static int chcr_offload_state(struct adapter *adap, 6385 enum cxgb4_netdev_tls_ops op_val) 6386{ 6387 switch (op_val) { 6388#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) 6389 case CXGB4_TLSDEV_OPS: 6390 if (!adap->uld[CXGB4_ULD_KTLS].handle) { 6391 dev_dbg(adap->pdev_dev, "ch_ktls driver is not loaded\n"); 6392 return -EOPNOTSUPP; 6393 } 6394 if (!adap->uld[CXGB4_ULD_KTLS].tlsdev_ops) { 6395 dev_dbg(adap->pdev_dev, 6396 "ch_ktls driver has no registered tlsdev_ops\n"); 6397 return -EOPNOTSUPP; 6398 } 6399 break; 6400#endif /* CONFIG_CHELSIO_TLS_DEVICE */ 6401#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) 6402 case CXGB4_XFRMDEV_OPS: 6403 if (!adap->uld[CXGB4_ULD_IPSEC].handle) { 6404 dev_dbg(adap->pdev_dev, "chipsec driver is not loaded\n"); 6405 return -EOPNOTSUPP; 6406 } 6407 if (!adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops) { 6408 dev_dbg(adap->pdev_dev, 6409 "chipsec driver has no registered xfrmdev_ops\n"); 6410 return -EOPNOTSUPP; 6411 } 6412 break; 6413#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ 6414 default: 6415 dev_dbg(adap->pdev_dev, 6416 "driver has no support for offload %d\n", op_val); 6417 return -EOPNOTSUPP; 6418 } 6419 6420 return 0; 6421} 6422 6423#endif /* CONFIG_CHELSIO_TLS_DEVICE || CONFIG_CHELSIO_IPSEC_INLINE */ 6424 6425#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) 6426 6427static int cxgb4_ktls_dev_add(struct net_device *netdev, struct sock *sk, 6428 enum tls_offload_ctx_dir direction, 6429 struct tls_crypto_info *crypto_info, 6430 u32 tcp_sn) 6431{ 6432 struct adapter *adap = netdev2adap(netdev); 6433 int ret; 6434 6435 mutex_lock(&uld_mutex); 6436 ret = chcr_offload_state(adap, CXGB4_TLSDEV_OPS); 6437 if (ret) 6438 goto out_unlock; 6439 6440 ret = cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_ENABLE); 6441 if (ret) 6442 goto out_unlock; 6443 6444 ret = adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_add(netdev, sk, 6445 direction, 6446 crypto_info, 6447 tcp_sn); 6448 /* if there is a failure, clear the refcount */ 6449 if (ret) 6450 cxgb4_set_ktls_feature(adap, 6451 FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); 6452out_unlock: 6453 mutex_unlock(&uld_mutex); 6454 return ret; 6455} 6456 6457static void cxgb4_ktls_dev_del(struct net_device *netdev, 6458 struct tls_context *tls_ctx, 6459 enum tls_offload_ctx_dir direction) 6460{ 6461 struct adapter *adap = netdev2adap(netdev); 6462 6463 mutex_lock(&uld_mutex); 6464 if (chcr_offload_state(adap, CXGB4_TLSDEV_OPS)) 6465 goto out_unlock; 6466 6467 adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_del(netdev, tls_ctx, 6468 direction); 6469 6470out_unlock: 6471 cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); 6472 mutex_unlock(&uld_mutex); 6473} 6474 6475static const struct tlsdev_ops cxgb4_ktls_ops = { 6476 .tls_dev_add = cxgb4_ktls_dev_add, 6477 .tls_dev_del = cxgb4_ktls_dev_del, 6478}; 6479#endif /* CONFIG_CHELSIO_TLS_DEVICE */ 6480 6481#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) 6482 6483static int cxgb4_xfrm_add_state(struct net_device *dev, 6484 struct xfrm_state *x, 6485 struct netlink_ext_ack *extack) 6486{ 6487 struct adapter *adap = netdev2adap(dev); 6488 int ret; 6489 6490 if (!mutex_trylock(&uld_mutex)) { 6491 NL_SET_ERR_MSG_MOD(extack, "crypto uld critical resource is under use"); 6492 return -EBUSY; 6493 } 6494 ret = chcr_offload_state(adap, CXGB4_XFRMDEV_OPS); 6495 if (ret) 6496 goto out_unlock; 6497 6498 ret = adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_add(dev, x, 6499 extack); 6500 6501out_unlock: 6502 mutex_unlock(&uld_mutex); 6503 6504 return ret; 6505} 6506 6507static void cxgb4_xfrm_del_state(struct net_device *dev, struct xfrm_state *x) 6508{ 6509 struct adapter *adap = netdev2adap(dev); 6510 6511 if (!mutex_trylock(&uld_mutex)) { 6512 dev_dbg(adap->pdev_dev, 6513 "crypto uld critical resource is under use\n"); 6514 return; 6515 } 6516 if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) 6517 goto out_unlock; 6518 6519 adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_delete(dev, x); 6520 6521out_unlock: 6522 mutex_unlock(&uld_mutex); 6523} 6524 6525static void cxgb4_xfrm_free_state(struct net_device *dev, struct xfrm_state *x) 6526{ 6527 struct adapter *adap = netdev2adap(dev); 6528 6529 if (!mutex_trylock(&uld_mutex)) { 6530 dev_dbg(adap->pdev_dev, 6531 "crypto uld critical resource is under use\n"); 6532 return; 6533 } 6534 if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) 6535 goto out_unlock; 6536 6537 adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_free(dev, x); 6538 6539out_unlock: 6540 mutex_unlock(&uld_mutex); 6541} 6542 6543static void cxgb4_advance_esn_state(struct xfrm_state *x) 6544{ 6545 struct adapter *adap = netdev2adap(x->xso.dev); 6546 6547 if (x->xso.dir != XFRM_DEV_OFFLOAD_IN) 6548 return; 6549 6550 if (!mutex_trylock(&uld_mutex)) { 6551 dev_dbg(adap->pdev_dev, 6552 "crypto uld critical resource is under use\n"); 6553 return; 6554 } 6555 if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS)) 6556 goto out_unlock; 6557 6558 adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_advance_esn(x); 6559 6560out_unlock: 6561 mutex_unlock(&uld_mutex); 6562} 6563 6564static const struct xfrmdev_ops cxgb4_xfrmdev_ops = { 6565 .xdo_dev_state_add = cxgb4_xfrm_add_state, 6566 .xdo_dev_state_delete = cxgb4_xfrm_del_state, 6567 .xdo_dev_state_free = cxgb4_xfrm_free_state, 6568 .xdo_dev_state_advance_esn = cxgb4_advance_esn_state, 6569}; 6570 6571#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ 6572 6573static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 6574{ 6575 struct net_device *netdev; 6576 struct adapter *adapter; 6577 static int adap_idx = 1; 6578 int s_qpp, qpp, num_seg; 6579 struct port_info *pi; 6580 enum chip_type chip; 6581 void __iomem *regs; 6582 int func, chip_ver; 6583 u16 device_id; 6584 int i, err; 6585 u32 whoami; 6586 6587 err = pci_request_regions(pdev, KBUILD_MODNAME); 6588 if (err) { 6589 /* Just info, some other driver may have claimed the device. */ 6590 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 6591 return err; 6592 } 6593 6594 err = pci_enable_device(pdev); 6595 if (err) { 6596 dev_err(&pdev->dev, "cannot enable PCI device\n"); 6597 goto out_release_regions; 6598 } 6599 6600 regs = pci_ioremap_bar(pdev, 0); 6601 if (!regs) { 6602 dev_err(&pdev->dev, "cannot map device registers\n"); 6603 err = -ENOMEM; 6604 goto out_disable_device; 6605 } 6606 6607 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 6608 if (!adapter) { 6609 err = -ENOMEM; 6610 goto out_unmap_bar0; 6611 } 6612 6613 adapter->regs = regs; 6614 err = t4_wait_dev_ready(regs); 6615 if (err < 0) 6616 goto out_free_adapter; 6617 6618 /* We control everything through one PF */ 6619 whoami = t4_read_reg(adapter, PL_WHOAMI_A); 6620 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); 6621 chip = t4_get_chip_type(adapter, CHELSIO_PCI_ID_VER(device_id)); 6622 if ((int)chip < 0) { 6623 dev_err(&pdev->dev, "Device %d is not supported\n", device_id); 6624 err = chip; 6625 goto out_free_adapter; 6626 } 6627 chip_ver = CHELSIO_CHIP_VERSION(chip); 6628 func = chip_ver <= CHELSIO_T5 ? 6629 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); 6630 6631 adapter->pdev = pdev; 6632 adapter->pdev_dev = &pdev->dev; 6633 adapter->name = pci_name(pdev); 6634 adapter->mbox = func; 6635 adapter->pf = func; 6636 adapter->params.chip = chip; 6637 adapter->adap_idx = adap_idx; 6638 adapter->msg_enable = DFLT_MSG_ENABLE; 6639 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 6640 (sizeof(struct mbox_cmd) * 6641 T4_OS_LOG_MBOX_CMDS), 6642 GFP_KERNEL); 6643 if (!adapter->mbox_log) { 6644 err = -ENOMEM; 6645 goto out_free_adapter; 6646 } 6647 spin_lock_init(&adapter->mbox_lock); 6648 INIT_LIST_HEAD(&adapter->mlist.list); 6649 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; 6650 pci_set_drvdata(pdev, adapter); 6651 6652 if (func != ent->driver_data) { 6653 pci_disable_device(pdev); 6654 pci_save_state(pdev); /* to restore SR-IOV later */ 6655 return 0; 6656 } 6657 6658 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 6659 if (err) { 6660 dev_err(&pdev->dev, "no usable DMA configuration\n"); 6661 goto out_free_adapter; 6662 } 6663 6664 pci_set_master(pdev); 6665 pci_save_state(pdev); 6666 adap_idx++; 6667 adapter->workq = create_singlethread_workqueue("cxgb4"); 6668 if (!adapter->workq) { 6669 err = -ENOMEM; 6670 goto out_free_adapter; 6671 } 6672 6673 /* PCI device has been enabled */ 6674 adapter->flags |= CXGB4_DEV_ENABLED; 6675 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 6676 6677 /* If possible, we use PCIe Relaxed Ordering Attribute to deliver 6678 * Ingress Packet Data to Free List Buffers in order to allow for 6679 * chipset performance optimizations between the Root Complex and 6680 * Memory Controllers. (Messages to the associated Ingress Queue 6681 * notifying new Packet Placement in the Free Lists Buffers will be 6682 * send without the Relaxed Ordering Attribute thus guaranteeing that 6683 * all preceding PCIe Transaction Layer Packets will be processed 6684 * first.) But some Root Complexes have various issues with Upstream 6685 * Transaction Layer Packets with the Relaxed Ordering Attribute set. 6686 * The PCIe devices which under the Root Complexes will be cleared the 6687 * Relaxed Ordering bit in the configuration space, So we check our 6688 * PCIe configuration space to see if it's flagged with advice against 6689 * using Relaxed Ordering. 6690 */ 6691 if (!pcie_relaxed_ordering_enabled(pdev)) 6692 adapter->flags |= CXGB4_ROOT_NO_RELAXED_ORDERING; 6693 6694 spin_lock_init(&adapter->stats_lock); 6695 spin_lock_init(&adapter->tid_release_lock); 6696 spin_lock_init(&adapter->win0_lock); 6697 6698 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 6699 INIT_WORK(&adapter->db_full_task, process_db_full); 6700 INIT_WORK(&adapter->db_drop_task, process_db_drop); 6701 INIT_WORK(&adapter->fatal_err_notify_task, notify_fatal_err); 6702 6703 err = t4_prep_adapter(adapter); 6704 if (err) 6705 goto out_free_adapter; 6706 6707 if (is_kdump_kernel()) { 6708 /* Collect hardware state and append to /proc/vmcore */ 6709 err = cxgb4_cudbg_vmcore_add_dump(adapter); 6710 if (err) { 6711 dev_warn(adapter->pdev_dev, 6712 "Fail collecting vmcore device dump, err: %d. Continuing\n", 6713 err); 6714 err = 0; 6715 } 6716 } 6717 6718 if (!is_t4(adapter->params.chip)) { 6719 s_qpp = (QUEUESPERPAGEPF0_S + 6720 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 6721 adapter->pf); 6722 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 6723 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 6724 num_seg = PAGE_SIZE / SEGMENT_SIZE; 6725 6726 /* Each segment size is 128B. Write coalescing is enabled only 6727 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 6728 * queue is less no of segments that can be accommodated in 6729 * a page size. 6730 */ 6731 if (qpp > num_seg) { 6732 dev_err(&pdev->dev, 6733 "Incorrect number of egress queues per page\n"); 6734 err = -EINVAL; 6735 goto out_free_adapter; 6736 } 6737 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 6738 pci_resource_len(pdev, 2)); 6739 if (!adapter->bar2) { 6740 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 6741 err = -ENOMEM; 6742 goto out_free_adapter; 6743 } 6744 } 6745 6746 setup_memwin(adapter); 6747 err = adap_init0(adapter, 0); 6748 if (err) 6749 goto out_unmap_bar; 6750 6751 setup_memwin_rdma(adapter); 6752 6753 /* configure SGE_STAT_CFG_A to read WC stats */ 6754 if (!is_t4(adapter->params.chip)) 6755 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | 6756 (is_t5(adapter->params.chip) ? STATMODE_V(0) : 6757 T6_STATMODE_V(0))); 6758 6759 /* Initialize hash mac addr list */ 6760 INIT_LIST_HEAD(&adapter->mac_hlist); 6761 6762 for_each_port(adapter, i) { 6763 /* For supporting MQPRIO Offload, need some extra 6764 * queues for each ETHOFLD TIDs. Keep it equal to 6765 * MAX_ATIDs for now. Once we connect to firmware 6766 * later and query the EOTID params, we'll come to 6767 * know the actual # of EOTIDs supported. 6768 */ 6769 netdev = alloc_etherdev_mq(sizeof(struct port_info), 6770 MAX_ETH_QSETS + MAX_ATIDS); 6771 if (!netdev) { 6772 err = -ENOMEM; 6773 goto out_free_dev; 6774 } 6775 6776 SET_NETDEV_DEV(netdev, &pdev->dev); 6777 6778 adapter->port[i] = netdev; 6779 pi = netdev_priv(netdev); 6780 pi->adapter = adapter; 6781 pi->xact_addr_filt = -1; 6782 pi->port_id = i; 6783 netdev->irq = pdev->irq; 6784 6785 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 6786 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 6787 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_GRO | 6788 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 6789 NETIF_F_HW_TC | NETIF_F_NTUPLE | NETIF_F_HIGHDMA; 6790 6791 if (chip_ver > CHELSIO_T5) { 6792 netdev->hw_enc_features |= NETIF_F_IP_CSUM | 6793 NETIF_F_IPV6_CSUM | 6794 NETIF_F_RXCSUM | 6795 NETIF_F_GSO_UDP_TUNNEL | 6796 NETIF_F_GSO_UDP_TUNNEL_CSUM | 6797 NETIF_F_TSO | NETIF_F_TSO6; 6798 6799 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 6800 NETIF_F_GSO_UDP_TUNNEL_CSUM | 6801 NETIF_F_HW_TLS_RECORD; 6802 6803 if (adapter->rawf_cnt) 6804 netdev->udp_tunnel_nic_info = &cxgb_udp_tunnels; 6805 } 6806 6807 netdev->features |= netdev->hw_features; 6808 netdev->vlan_features = netdev->features & VLAN_FEAT; 6809#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) 6810 if (pi->adapter->params.crypto & FW_CAPS_CONFIG_TLS_HW) { 6811 netdev->hw_features |= NETIF_F_HW_TLS_TX; 6812 netdev->tlsdev_ops = &cxgb4_ktls_ops; 6813 /* initialize the refcount */ 6814 refcount_set(&pi->adapter->chcr_ktls.ktls_refcount, 0); 6815 } 6816#endif /* CONFIG_CHELSIO_TLS_DEVICE */ 6817#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) 6818 if (pi->adapter->params.crypto & FW_CAPS_CONFIG_IPSEC_INLINE) { 6819 netdev->hw_enc_features |= NETIF_F_HW_ESP; 6820 netdev->features |= NETIF_F_HW_ESP; 6821 netdev->xfrmdev_ops = &cxgb4_xfrmdev_ops; 6822 } 6823#endif /* CONFIG_CHELSIO_IPSEC_INLINE */ 6824 6825 netdev->priv_flags |= IFF_UNICAST_FLT; 6826 6827 /* MTU range: 81 - 9600 */ 6828 netdev->min_mtu = 81; /* accommodate SACK */ 6829 netdev->max_mtu = MAX_MTU; 6830 6831 netdev->netdev_ops = &cxgb4_netdev_ops; 6832#ifdef CONFIG_CHELSIO_T4_DCB 6833 netdev->dcbnl_ops = &cxgb4_dcb_ops; 6834 cxgb4_dcb_state_init(netdev); 6835 cxgb4_dcb_version_init(netdev); 6836#endif 6837 cxgb4_set_ethtool_ops(netdev); 6838 } 6839 6840 cxgb4_init_ethtool_dump(adapter); 6841 6842 pci_set_drvdata(pdev, adapter); 6843 6844 if (adapter->flags & CXGB4_FW_OK) { 6845 err = t4_port_init(adapter, func, func, 0); 6846 if (err) 6847 goto out_free_dev; 6848 } else if (adapter->params.nports == 1) { 6849 /* If we don't have a connection to the firmware -- possibly 6850 * because of an error -- grab the raw VPD parameters so we 6851 * can set the proper MAC Address on the debug network 6852 * interface that we've created. 6853 */ 6854 u8 hw_addr[ETH_ALEN]; 6855 u8 *na = adapter->params.vpd.na; 6856 6857 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); 6858 if (!err) { 6859 for (i = 0; i < ETH_ALEN; i++) 6860 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 6861 hex2val(na[2 * i + 1])); 6862 t4_set_hw_addr(adapter, 0, hw_addr); 6863 } 6864 } 6865 6866 if (!(adapter->flags & CXGB4_FW_OK)) 6867 goto fw_attach_fail; 6868 6869 /* Configure queues and allocate tables now, they can be needed as 6870 * soon as the first register_netdev completes. 6871 */ 6872 err = cfg_queues(adapter); 6873 if (err) 6874 goto out_free_dev; 6875 6876 adapter->smt = t4_init_smt(); 6877 if (!adapter->smt) { 6878 /* We tolerate a lack of SMT, giving up some functionality */ 6879 dev_warn(&pdev->dev, "could not allocate SMT, continuing\n"); 6880 } 6881 6882 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); 6883 if (!adapter->l2t) { 6884 /* We tolerate a lack of L2T, giving up some functionality */ 6885 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 6886 adapter->params.offload = 0; 6887 } 6888 6889#if IS_ENABLED(CONFIG_IPV6) 6890 if (chip_ver <= CHELSIO_T5 && 6891 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { 6892 /* CLIP functionality is not present in hardware, 6893 * hence disable all offload features 6894 */ 6895 dev_warn(&pdev->dev, 6896 "CLIP not enabled in hardware, continuing\n"); 6897 adapter->params.offload = 0; 6898 } else { 6899 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 6900 adapter->clipt_end); 6901 if (!adapter->clipt) { 6902 /* We tolerate a lack of clip_table, giving up 6903 * some functionality 6904 */ 6905 dev_warn(&pdev->dev, 6906 "could not allocate Clip table, continuing\n"); 6907 adapter->params.offload = 0; 6908 } 6909 } 6910#endif 6911 6912 for_each_port(adapter, i) { 6913 pi = adap2pinfo(adapter, i); 6914 pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls); 6915 if (!pi->sched_tbl) 6916 dev_warn(&pdev->dev, 6917 "could not activate scheduling on port %d\n", 6918 i); 6919 } 6920 6921 if (is_offload(adapter) || is_hashfilter(adapter)) { 6922 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { 6923 u32 v; 6924 6925 v = t4_read_reg(adapter, LE_DB_HASH_CONFIG_A); 6926 if (chip_ver <= CHELSIO_T5) { 6927 adapter->tids.nhash = 1 << HASHTIDSIZE_G(v); 6928 v = t4_read_reg(adapter, LE_DB_TID_HASHBASE_A); 6929 adapter->tids.hash_base = v / 4; 6930 } else { 6931 adapter->tids.nhash = HASHTBLSIZE_G(v) << 3; 6932 v = t4_read_reg(adapter, 6933 T6_LE_DB_HASH_TID_BASE_A); 6934 adapter->tids.hash_base = v; 6935 } 6936 } 6937 } 6938 6939 if (tid_init(&adapter->tids) < 0) { 6940 dev_warn(&pdev->dev, "could not allocate TID table, " 6941 "continuing\n"); 6942 adapter->params.offload = 0; 6943 } else { 6944 adapter->tc_u32 = cxgb4_init_tc_u32(adapter); 6945 if (!adapter->tc_u32) 6946 dev_warn(&pdev->dev, 6947 "could not offload tc u32, continuing\n"); 6948 6949 if (cxgb4_init_tc_flower(adapter)) 6950 dev_warn(&pdev->dev, 6951 "could not offload tc flower, continuing\n"); 6952 6953 if (cxgb4_init_tc_mqprio(adapter)) 6954 dev_warn(&pdev->dev, 6955 "could not offload tc mqprio, continuing\n"); 6956 6957 if (cxgb4_init_tc_matchall(adapter)) 6958 dev_warn(&pdev->dev, 6959 "could not offload tc matchall, continuing\n"); 6960 if (cxgb4_init_ethtool_filters(adapter)) 6961 dev_warn(&pdev->dev, 6962 "could not initialize ethtool filters, continuing\n"); 6963 } 6964 6965 /* See what interrupts we'll be using */ 6966 if (msi > 1 && enable_msix(adapter) == 0) 6967 adapter->flags |= CXGB4_USING_MSIX; 6968 else if (msi > 0 && pci_enable_msi(pdev) == 0) { 6969 adapter->flags |= CXGB4_USING_MSI; 6970 if (msi > 1) 6971 free_msix_info(adapter); 6972 } 6973 6974 /* check for PCI Express bandwidth capabiltites */ 6975 pcie_print_link_status(pdev); 6976 6977 cxgb4_init_mps_ref_entries(adapter); 6978 6979 err = init_rss(adapter); 6980 if (err) 6981 goto out_free_dev; 6982 6983 err = setup_non_data_intr(adapter); 6984 if (err) { 6985 dev_err(adapter->pdev_dev, 6986 "Non Data interrupt allocation failed, err: %d\n", err); 6987 goto out_free_dev; 6988 } 6989 6990 err = setup_fw_sge_queues(adapter); 6991 if (err) { 6992 dev_err(adapter->pdev_dev, 6993 "FW sge queue allocation failed, err %d", err); 6994 goto out_free_dev; 6995 } 6996 6997fw_attach_fail: 6998 /* 6999 * The card is now ready to go. If any errors occur during device 7000 * registration we do not fail the whole card but rather proceed only 7001 * with the ports we manage to register successfully. However we must 7002 * register at least one net device. 7003 */ 7004 for_each_port(adapter, i) { 7005 pi = adap2pinfo(adapter, i); 7006 adapter->port[i]->dev_port = pi->lport; 7007 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 7008 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 7009 7010 netif_carrier_off(adapter->port[i]); 7011 7012 err = register_netdev(adapter->port[i]); 7013 if (err) 7014 break; 7015 adapter->chan_map[pi->tx_chan] = i; 7016 print_port_info(adapter->port[i]); 7017 } 7018 if (i == 0) { 7019 dev_err(&pdev->dev, "could not register any net devices\n"); 7020 goto out_free_dev; 7021 } 7022 if (err) { 7023 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 7024 err = 0; 7025 } 7026 7027 if (cxgb4_debugfs_root) { 7028 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 7029 cxgb4_debugfs_root); 7030 setup_debugfs(adapter); 7031 } 7032 7033 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 7034 pdev->needs_freset = 1; 7035 7036 if (is_uld(adapter)) 7037 cxgb4_uld_enable(adapter); 7038 7039 if (!is_t4(adapter->params.chip)) 7040 cxgb4_ptp_init(adapter); 7041 7042 if (IS_REACHABLE(CONFIG_THERMAL) && 7043 !is_t4(adapter->params.chip) && (adapter->flags & CXGB4_FW_OK)) 7044 cxgb4_thermal_init(adapter); 7045 7046 print_adapter_info(adapter); 7047 return 0; 7048 7049 out_free_dev: 7050 t4_free_sge_resources(adapter); 7051 free_some_resources(adapter); 7052 if (adapter->flags & CXGB4_USING_MSIX) 7053 free_msix_info(adapter); 7054 if (adapter->num_uld || adapter->num_ofld_uld) 7055 t4_uld_mem_free(adapter); 7056 out_unmap_bar: 7057 if (!is_t4(adapter->params.chip)) 7058 iounmap(adapter->bar2); 7059 out_free_adapter: 7060 if (adapter->workq) 7061 destroy_workqueue(adapter->workq); 7062 7063 kfree(adapter->mbox_log); 7064 kfree(adapter); 7065 out_unmap_bar0: 7066 iounmap(regs); 7067 out_disable_device: 7068 pci_disable_device(pdev); 7069 out_release_regions: 7070 pci_release_regions(pdev); 7071 return err; 7072} 7073 7074static void remove_one(struct pci_dev *pdev) 7075{ 7076 struct adapter *adapter = pci_get_drvdata(pdev); 7077 struct hash_mac_addr *entry, *tmp; 7078 7079 if (!adapter) { 7080 pci_release_regions(pdev); 7081 return; 7082 } 7083 7084 /* If we allocated filters, free up state associated with any 7085 * valid filters ... 7086 */ 7087 clear_all_filters(adapter); 7088 7089 adapter->flags |= CXGB4_SHUTTING_DOWN; 7090 7091 if (adapter->pf == 4) { 7092 int i; 7093 7094 /* Tear down per-adapter Work Queue first since it can contain 7095 * references to our adapter data structure. 7096 */ 7097 destroy_workqueue(adapter->workq); 7098 7099 detach_ulds(adapter); 7100 7101 for_each_port(adapter, i) 7102 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 7103 unregister_netdev(adapter->port[i]); 7104 7105 t4_uld_clean_up(adapter); 7106 7107 adap_free_hma_mem(adapter); 7108 7109 disable_interrupts(adapter); 7110 7111 cxgb4_free_mps_ref_entries(adapter); 7112 7113 debugfs_remove_recursive(adapter->debugfs_root); 7114 7115 if (!is_t4(adapter->params.chip)) 7116 cxgb4_ptp_stop(adapter); 7117 if (IS_REACHABLE(CONFIG_THERMAL)) 7118 cxgb4_thermal_remove(adapter); 7119 7120 if (adapter->flags & CXGB4_FULL_INIT_DONE) 7121 cxgb_down(adapter); 7122 7123 if (adapter->flags & CXGB4_USING_MSIX) 7124 free_msix_info(adapter); 7125 if (adapter->num_uld || adapter->num_ofld_uld) 7126 t4_uld_mem_free(adapter); 7127 free_some_resources(adapter); 7128 list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, 7129 list) { 7130 list_del(&entry->list); 7131 kfree(entry); 7132 } 7133 7134#if IS_ENABLED(CONFIG_IPV6) 7135 t4_cleanup_clip_tbl(adapter); 7136#endif 7137 if (!is_t4(adapter->params.chip)) 7138 iounmap(adapter->bar2); 7139 } 7140#ifdef CONFIG_PCI_IOV 7141 else { 7142 cxgb4_iov_configure(adapter->pdev, 0); 7143 } 7144#endif 7145 iounmap(adapter->regs); 7146 if ((adapter->flags & CXGB4_DEV_ENABLED)) { 7147 pci_disable_device(pdev); 7148 adapter->flags &= ~CXGB4_DEV_ENABLED; 7149 } 7150 pci_release_regions(pdev); 7151 kfree(adapter->mbox_log); 7152 synchronize_rcu(); 7153 kfree(adapter); 7154} 7155 7156/* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt 7157 * delivery. This is essentially a stripped down version of the PCI remove() 7158 * function where we do the minimal amount of work necessary to shutdown any 7159 * further activity. 7160 */ 7161static void shutdown_one(struct pci_dev *pdev) 7162{ 7163 struct adapter *adapter = pci_get_drvdata(pdev); 7164 7165 /* As with remove_one() above (see extended comment), we only want do 7166 * do cleanup on PCI Devices which went all the way through init_one() 7167 * ... 7168 */ 7169 if (!adapter) { 7170 pci_release_regions(pdev); 7171 return; 7172 } 7173 7174 adapter->flags |= CXGB4_SHUTTING_DOWN; 7175 7176 if (adapter->pf == 4) { 7177 int i; 7178 7179 for_each_port(adapter, i) 7180 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 7181 cxgb_close(adapter->port[i]); 7182 7183 rtnl_lock(); 7184 cxgb4_mqprio_stop_offload(adapter); 7185 rtnl_unlock(); 7186 7187 if (is_uld(adapter)) { 7188 detach_ulds(adapter); 7189 t4_uld_clean_up(adapter); 7190 } 7191 7192 disable_interrupts(adapter); 7193 disable_msi(adapter); 7194 7195 t4_sge_stop(adapter); 7196 if (adapter->flags & CXGB4_FW_OK) 7197 t4_fw_bye(adapter, adapter->mbox); 7198 } 7199} 7200 7201static struct pci_driver cxgb4_driver = { 7202 .name = KBUILD_MODNAME, 7203 .id_table = cxgb4_pci_tbl, 7204 .probe = init_one, 7205 .remove = remove_one, 7206 .shutdown = shutdown_one, 7207#ifdef CONFIG_PCI_IOV 7208 .sriov_configure = cxgb4_iov_configure, 7209#endif 7210 .err_handler = &cxgb4_eeh, 7211}; 7212 7213static int __init cxgb4_init_module(void) 7214{ 7215 int ret; 7216 7217 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 7218 7219 ret = pci_register_driver(&cxgb4_driver); 7220 if (ret < 0) 7221 goto err_pci; 7222 7223#if IS_ENABLED(CONFIG_IPV6) 7224 if (!inet6addr_registered) { 7225 ret = register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 7226 if (ret) 7227 pci_unregister_driver(&cxgb4_driver); 7228 else 7229 inet6addr_registered = true; 7230 } 7231#endif 7232 7233 if (ret == 0) 7234 return ret; 7235 7236err_pci: 7237 debugfs_remove(cxgb4_debugfs_root); 7238 7239 return ret; 7240} 7241 7242static void __exit cxgb4_cleanup_module(void) 7243{ 7244#if IS_ENABLED(CONFIG_IPV6) 7245 if (inet6addr_registered) { 7246 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 7247 inet6addr_registered = false; 7248 } 7249#endif 7250 pci_unregister_driver(&cxgb4_driver); 7251 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 7252} 7253 7254module_init(cxgb4_init_module); 7255module_exit(cxgb4_cleanup_module);
~/
Read-only file preview.