FILE
myri10ge.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
1/************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43#include <linux/tcp.h> 44#include <linux/netdevice.h> 45#include <linux/skbuff.h> 46#include <linux/string.h> 47#include <linux/module.h> 48#include <linux/pci.h> 49#include <linux/dma-mapping.h> 50#include <linux/etherdevice.h> 51#include <linux/if_ether.h> 52#include <linux/if_vlan.h> 53#include <linux/dca.h> 54#include <linux/ip.h> 55#include <linux/inet.h> 56#include <linux/in.h> 57#include <linux/ethtool.h> 58#include <linux/firmware.h> 59#include <linux/delay.h> 60#include <linux/timer.h> 61#include <linux/vmalloc.h> 62#include <linux/crc32.h> 63#include <linux/moduleparam.h> 64#include <linux/io.h> 65#include <linux/log2.h> 66#include <linux/slab.h> 67#include <linux/prefetch.h> 68#include <net/checksum.h> 69#include <net/gso.h> 70#include <net/ip.h> 71#include <net/tcp.h> 72#include <asm/byteorder.h> 73#include <asm/processor.h> 74 75#include "myri10ge_mcp.h" 76#include "myri10ge_mcp_gen_header.h" 77 78#define MYRI10GE_VERSION_STR "1.5.3-1.534" 79 80MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 81MODULE_AUTHOR("Maintainer: help@myri.com"); 82MODULE_VERSION(MYRI10GE_VERSION_STR); 83MODULE_LICENSE("Dual BSD/GPL"); 84 85#define MYRI10GE_MAX_ETHER_MTU 9014 86 87#define MYRI10GE_ETH_STOPPED 0 88#define MYRI10GE_ETH_STOPPING 1 89#define MYRI10GE_ETH_STARTING 2 90#define MYRI10GE_ETH_RUNNING 3 91#define MYRI10GE_ETH_OPEN_FAILED 4 92 93#define MYRI10GE_EEPROM_STRINGS_SIZE 256 94#define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 95 96#define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 97#define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 98 99#define MYRI10GE_ALLOC_ORDER 0 100#define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 101#define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 102 103#define MYRI10GE_MAX_SLICES 32 104 105struct myri10ge_rx_buffer_state { 106 struct page *page; 107 int page_offset; 108 DEFINE_DMA_UNMAP_ADDR(bus); 109 DEFINE_DMA_UNMAP_LEN(len); 110}; 111 112struct myri10ge_tx_buffer_state { 113 struct sk_buff *skb; 114 int last; 115 DEFINE_DMA_UNMAP_ADDR(bus); 116 DEFINE_DMA_UNMAP_LEN(len); 117}; 118 119struct myri10ge_cmd { 120 u32 data0; 121 u32 data1; 122 u32 data2; 123}; 124 125struct myri10ge_rx_buf { 126 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 127 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 128 struct myri10ge_rx_buffer_state *info; 129 struct page *page; 130 dma_addr_t bus; 131 int page_offset; 132 int cnt; 133 int fill_cnt; 134 int alloc_fail; 135 int mask; /* number of rx slots -1 */ 136 int watchdog_needed; 137}; 138 139struct myri10ge_tx_buf { 140 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 141 __be32 __iomem *send_go; /* "go" doorbell ptr */ 142 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 143 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 144 char *req_bytes; 145 struct myri10ge_tx_buffer_state *info; 146 int mask; /* number of transmit slots -1 */ 147 int req ____cacheline_aligned; /* transmit slots submitted */ 148 int pkt_start; /* packets started */ 149 int stop_queue; 150 int linearized; 151 int done ____cacheline_aligned; /* transmit slots completed */ 152 int pkt_done; /* packets completed */ 153 int wake_queue; 154 int queue_active; 155}; 156 157struct myri10ge_rx_done { 158 struct mcp_slot *entry; 159 dma_addr_t bus; 160 int cnt; 161 int idx; 162}; 163 164struct myri10ge_slice_netstats { 165 unsigned long rx_packets; 166 unsigned long tx_packets; 167 unsigned long rx_bytes; 168 unsigned long tx_bytes; 169 unsigned long rx_dropped; 170 unsigned long tx_dropped; 171}; 172 173struct myri10ge_slice_state { 174 struct myri10ge_tx_buf tx; /* transmit ring */ 175 struct myri10ge_rx_buf rx_small; 176 struct myri10ge_rx_buf rx_big; 177 struct myri10ge_rx_done rx_done; 178 struct net_device *dev; 179 struct napi_struct napi; 180 struct myri10ge_priv *mgp; 181 struct myri10ge_slice_netstats stats; 182 __be32 __iomem *irq_claim; 183 struct mcp_irq_data *fw_stats; 184 dma_addr_t fw_stats_bus; 185 int watchdog_tx_done; 186 int watchdog_tx_req; 187 int watchdog_rx_done; 188 int stuck; 189#ifdef CONFIG_MYRI10GE_DCA 190 int cached_dca_tag; 191 int cpu; 192 __be32 __iomem *dca_tag; 193#endif 194 char irq_desc[32]; 195}; 196 197struct myri10ge_priv { 198 struct myri10ge_slice_state *ss; 199 int tx_boundary; /* boundary transmits cannot cross */ 200 int num_slices; 201 int running; /* running? */ 202 int small_bytes; 203 int big_bytes; 204 int max_intr_slots; 205 struct net_device *dev; 206 u8 __iomem *sram; 207 int sram_size; 208 unsigned long board_span; 209 unsigned long iomem_base; 210 __be32 __iomem *irq_deassert; 211 char *mac_addr_string; 212 struct mcp_cmd_response *cmd; 213 dma_addr_t cmd_bus; 214 struct pci_dev *pdev; 215 int msi_enabled; 216 int msix_enabled; 217 struct msix_entry *msix_vectors; 218#ifdef CONFIG_MYRI10GE_DCA 219 int dca_enabled; 220 int relaxed_order; 221#endif 222 u32 link_state; 223 unsigned int rdma_tags_available; 224 int intr_coal_delay; 225 __be32 __iomem *intr_coal_delay_ptr; 226 int wc_cookie; 227 int down_cnt; 228 wait_queue_head_t down_wq; 229 struct work_struct watchdog_work; 230 struct timer_list watchdog_timer; 231 int watchdog_resets; 232 int watchdog_pause; 233 int pause; 234 bool fw_name_allocated; 235 char *fw_name; 236 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 237 char *product_code_string; 238 char fw_version[128]; 239 int fw_ver_major; 240 int fw_ver_minor; 241 int fw_ver_tiny; 242 int adopted_rx_filter_bug; 243 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 244 unsigned long serial_number; 245 int vendor_specific_offset; 246 int fw_multicast_support; 247 u32 features; 248 u32 max_tso6; 249 u32 read_dma; 250 u32 write_dma; 251 u32 read_write_dma; 252 u32 link_changes; 253 u32 msg_enable; 254 unsigned int board_number; 255 int rebooted; 256}; 257 258static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 259static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 260static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 261static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 262MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 263MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 264MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 265MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 266 267/* Careful: must be accessed under kernel_param_lock() */ 268static char *myri10ge_fw_name = NULL; 269module_param(myri10ge_fw_name, charp, 0644); 270MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 271 272#define MYRI10GE_MAX_BOARDS 8 273static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 274 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 275module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 276 0444); 277MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 278 279static int myri10ge_ecrc_enable = 1; 280module_param(myri10ge_ecrc_enable, int, 0444); 281MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 282 283static int myri10ge_small_bytes = -1; /* -1 == auto */ 284module_param(myri10ge_small_bytes, int, 0644); 285MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 286 287static int myri10ge_msi = 1; /* enable msi by default */ 288module_param(myri10ge_msi, int, 0644); 289MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 290 291static int myri10ge_intr_coal_delay = 75; 292module_param(myri10ge_intr_coal_delay, int, 0444); 293MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 294 295static int myri10ge_flow_control = 1; 296module_param(myri10ge_flow_control, int, 0444); 297MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 298 299static int myri10ge_deassert_wait = 1; 300module_param(myri10ge_deassert_wait, int, 0644); 301MODULE_PARM_DESC(myri10ge_deassert_wait, 302 "Wait when deasserting legacy interrupts"); 303 304static int myri10ge_force_firmware = 0; 305module_param(myri10ge_force_firmware, int, 0444); 306MODULE_PARM_DESC(myri10ge_force_firmware, 307 "Force firmware to assume aligned completions"); 308 309static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 310module_param(myri10ge_initial_mtu, int, 0444); 311MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 312 313static int myri10ge_napi_weight = 64; 314module_param(myri10ge_napi_weight, int, 0444); 315MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 316 317static int myri10ge_watchdog_timeout = 1; 318module_param(myri10ge_watchdog_timeout, int, 0444); 319MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 320 321static int myri10ge_max_irq_loops = 1048576; 322module_param(myri10ge_max_irq_loops, int, 0444); 323MODULE_PARM_DESC(myri10ge_max_irq_loops, 324 "Set stuck legacy IRQ detection threshold"); 325 326#define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 327 328static int myri10ge_debug = -1; /* defaults above */ 329module_param(myri10ge_debug, int, 0); 330MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 331 332static int myri10ge_fill_thresh = 256; 333module_param(myri10ge_fill_thresh, int, 0644); 334MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 335 336static int myri10ge_reset_recover = 1; 337 338static int myri10ge_max_slices = 1; 339module_param(myri10ge_max_slices, int, 0444); 340MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 341 342static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 343module_param(myri10ge_rss_hash, int, 0444); 344MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 345 346static int myri10ge_dca = 1; 347module_param(myri10ge_dca, int, 0444); 348MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 349 350#define MYRI10GE_FW_OFFSET 1024*1024 351#define MYRI10GE_HIGHPART_TO_U32(X) \ 352(sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 353#define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 354 355#define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 356 357static void myri10ge_set_multicast_list(struct net_device *dev); 358static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 359 struct net_device *dev); 360 361static inline void put_be32(__be32 val, __be32 __iomem * p) 362{ 363 __raw_writel((__force __u32) val, (__force void __iomem *)p); 364} 365 366static void myri10ge_get_stats(struct net_device *dev, 367 struct rtnl_link_stats64 *stats); 368 369static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 370{ 371 if (mgp->fw_name_allocated) 372 kfree(mgp->fw_name); 373 mgp->fw_name = name; 374 mgp->fw_name_allocated = allocated; 375} 376 377static int 378myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 379 struct myri10ge_cmd *data, int atomic) 380{ 381 struct mcp_cmd *buf; 382 char buf_bytes[sizeof(*buf) + 8]; 383 struct mcp_cmd_response *response = mgp->cmd; 384 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 385 u32 dma_low, dma_high, result, value; 386 int sleep_total = 0; 387 388 /* ensure buf is aligned to 8 bytes */ 389 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 390 391 buf->data0 = htonl(data->data0); 392 buf->data1 = htonl(data->data1); 393 buf->data2 = htonl(data->data2); 394 buf->cmd = htonl(cmd); 395 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 396 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 397 398 buf->response_addr.low = htonl(dma_low); 399 buf->response_addr.high = htonl(dma_high); 400 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 401 mb(); 402 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 403 404 /* wait up to 15ms. Longest command is the DMA benchmark, 405 * which is capped at 5ms, but runs from a timeout handler 406 * that runs every 7.8ms. So a 15ms timeout leaves us with 407 * a 2.2ms margin 408 */ 409 if (atomic) { 410 /* if atomic is set, do not sleep, 411 * and try to get the completion quickly 412 * (1ms will be enough for those commands) */ 413 for (sleep_total = 0; 414 sleep_total < 1000 && 415 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 416 sleep_total += 10) { 417 udelay(10); 418 mb(); 419 } 420 } else { 421 /* use msleep for most command */ 422 for (sleep_total = 0; 423 sleep_total < 15 && 424 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 425 sleep_total++) 426 msleep(1); 427 } 428 429 result = ntohl(response->result); 430 value = ntohl(response->data); 431 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 432 if (result == 0) { 433 data->data0 = value; 434 return 0; 435 } else if (result == MXGEFW_CMD_UNKNOWN) { 436 return -ENOSYS; 437 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 438 return -E2BIG; 439 } else if (result == MXGEFW_CMD_ERROR_RANGE && 440 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 441 (data-> 442 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 443 0) { 444 return -ERANGE; 445 } else { 446 dev_err(&mgp->pdev->dev, 447 "command %d failed, result = %d\n", 448 cmd, result); 449 return -ENXIO; 450 } 451 } 452 453 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 454 cmd, result); 455 return -EAGAIN; 456} 457 458/* 459 * The eeprom strings on the lanaiX have the format 460 * SN=x\0 461 * MAC=x:x:x:x:x:x\0 462 * PT:ddd mmm xx xx:xx:xx xx\0 463 * PV:ddd mmm xx xx:xx:xx xx\0 464 */ 465static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 466{ 467 char *ptr, *limit; 468 int i; 469 470 ptr = mgp->eeprom_strings; 471 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 472 473 while (*ptr != '\0' && ptr < limit) { 474 if (memcmp(ptr, "MAC=", 4) == 0) { 475 ptr += 4; 476 mgp->mac_addr_string = ptr; 477 for (i = 0; i < 6; i++) { 478 if ((ptr + 2) > limit) 479 goto abort; 480 mgp->mac_addr[i] = 481 simple_strtoul(ptr, &ptr, 16); 482 ptr += 1; 483 } 484 } 485 if (memcmp(ptr, "PC=", 3) == 0) { 486 ptr += 3; 487 mgp->product_code_string = ptr; 488 } 489 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 490 ptr += 3; 491 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 492 } 493 while (ptr < limit && *ptr++) ; 494 } 495 496 return 0; 497 498abort: 499 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 500 return -ENXIO; 501} 502 503/* 504 * Enable or disable periodic RDMAs from the host to make certain 505 * chipsets resend dropped PCIe messages 506 */ 507 508static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 509{ 510 char __iomem *submit; 511 __be32 buf[16] __attribute__ ((__aligned__(8))); 512 u32 dma_low, dma_high; 513 int i; 514 515 /* clear confirmation addr */ 516 mgp->cmd->data = 0; 517 mb(); 518 519 /* send a rdma command to the PCIe engine, and wait for the 520 * response in the confirmation address. The firmware should 521 * write a -1 there to indicate it is alive and well 522 */ 523 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 524 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 525 526 buf[0] = htonl(dma_high); /* confirm addr MSW */ 527 buf[1] = htonl(dma_low); /* confirm addr LSW */ 528 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 529 buf[3] = htonl(dma_high); /* dummy addr MSW */ 530 buf[4] = htonl(dma_low); /* dummy addr LSW */ 531 buf[5] = htonl(enable); /* enable? */ 532 533 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 534 535 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 536 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 537 msleep(1); 538 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 539 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 540 (enable ? "enable" : "disable")); 541} 542 543static int 544myri10ge_validate_firmware(struct myri10ge_priv *mgp, 545 struct mcp_gen_header *hdr) 546{ 547 struct device *dev = &mgp->pdev->dev; 548 549 /* check firmware type */ 550 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 551 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 552 return -EINVAL; 553 } 554 555 /* save firmware version for ethtool */ 556 strscpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569} 570 571static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572{ 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if (request_firmware(&fw, mgp->fw_name, dev) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636abort_with_fw: 637 release_firmware(fw); 638 639abort_with_nothing: 640 return status; 641} 642 643static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644{ 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682} 683 684static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685{ 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 cmd.data0 = 0, 692 cmd.data1 = 0, 693 cmd.data2 = 0, 694 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 695 &cmd, 0); 696 if (status == 0) { 697 mgp->max_tso6 = cmd.data0; 698 mgp->features |= NETIF_F_TSO6; 699 } 700 701 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 702 if (status != 0) { 703 dev_err(&mgp->pdev->dev, 704 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 705 return -ENXIO; 706 } 707 708 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 709 710 return 0; 711} 712 713static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 714{ 715 char __iomem *submit; 716 __be32 buf[16] __attribute__ ((__aligned__(8))); 717 u32 dma_low, dma_high, size; 718 int status, i; 719 720 size = 0; 721 status = myri10ge_load_hotplug_firmware(mgp, &size); 722 if (status) { 723 if (!adopt) 724 return status; 725 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 726 727 /* Do not attempt to adopt firmware if there 728 * was a bad crc */ 729 if (status == -EIO) 730 return status; 731 732 status = myri10ge_adopt_running_firmware(mgp); 733 if (status != 0) { 734 dev_err(&mgp->pdev->dev, 735 "failed to adopt running firmware\n"); 736 return status; 737 } 738 dev_info(&mgp->pdev->dev, 739 "Successfully adopted running firmware\n"); 740 if (mgp->tx_boundary == 4096) { 741 dev_warn(&mgp->pdev->dev, 742 "Using firmware currently running on NIC" 743 ". For optimal\n"); 744 dev_warn(&mgp->pdev->dev, 745 "performance consider loading optimized " 746 "firmware\n"); 747 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 748 } 749 750 set_fw_name(mgp, "adopted", false); 751 mgp->tx_boundary = 2048; 752 myri10ge_dummy_rdma(mgp, 1); 753 status = myri10ge_get_firmware_capabilities(mgp); 754 return status; 755 } 756 757 /* clear confirmation addr */ 758 mgp->cmd->data = 0; 759 mb(); 760 761 /* send a reload command to the bootstrap MCP, and wait for the 762 * response in the confirmation address. The firmware should 763 * write a -1 there to indicate it is alive and well 764 */ 765 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 766 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 767 768 buf[0] = htonl(dma_high); /* confirm addr MSW */ 769 buf[1] = htonl(dma_low); /* confirm addr LSW */ 770 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 771 772 /* FIX: All newest firmware should un-protect the bottom of 773 * the sram before handoff. However, the very first interfaces 774 * do not. Therefore the handoff copy must skip the first 8 bytes 775 */ 776 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 777 buf[4] = htonl(size - 8); /* length of code */ 778 buf[5] = htonl(8); /* where to copy to */ 779 buf[6] = htonl(0); /* where to jump to */ 780 781 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 782 783 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 784 mb(); 785 msleep(1); 786 mb(); 787 i = 0; 788 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 789 msleep(1 << i); 790 i++; 791 } 792 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 793 dev_err(&mgp->pdev->dev, "handoff failed\n"); 794 return -ENXIO; 795 } 796 myri10ge_dummy_rdma(mgp, 1); 797 status = myri10ge_get_firmware_capabilities(mgp); 798 799 return status; 800} 801 802static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, 803 const u8 * addr) 804{ 805 struct myri10ge_cmd cmd; 806 int status; 807 808 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 809 | (addr[2] << 8) | addr[3]); 810 811 cmd.data1 = ((addr[4] << 8) | (addr[5])); 812 cmd.data2 = 0; 813 814 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 815 return status; 816} 817 818static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 819{ 820 struct myri10ge_cmd cmd; 821 int status, ctl; 822 823 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 824 cmd.data0 = 0, 825 cmd.data1 = 0, 826 cmd.data2 = 0, 827 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 828 829 if (status) { 830 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 831 return status; 832 } 833 mgp->pause = pause; 834 return 0; 835} 836 837static void 838myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 839{ 840 struct myri10ge_cmd cmd; 841 int status, ctl; 842 843 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 844 cmd.data0 = 0; 845 cmd.data1 = 0; 846 cmd.data2 = 0; 847 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 848 if (status) 849 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 850} 851 852static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 853{ 854 struct myri10ge_cmd cmd; 855 int status; 856 u32 len; 857 struct page *dmatest_page; 858 dma_addr_t dmatest_bus; 859 char *test = " "; 860 861 dmatest_page = alloc_page(GFP_KERNEL); 862 if (!dmatest_page) 863 return -ENOMEM; 864 dmatest_bus = dma_map_page(&mgp->pdev->dev, dmatest_page, 0, 865 PAGE_SIZE, DMA_BIDIRECTIONAL); 866 if (unlikely(dma_mapping_error(&mgp->pdev->dev, dmatest_bus))) { 867 __free_page(dmatest_page); 868 return -ENOMEM; 869 } 870 871 /* Run a small DMA test. 872 * The magic multipliers to the length tell the firmware 873 * to do DMA read, write, or read+write tests. The 874 * results are returned in cmd.data0. The upper 16 875 * bits or the return is the number of transfers completed. 876 * The lower 16 bits is the time in 0.5us ticks that the 877 * transfers took to complete. 878 */ 879 880 len = mgp->tx_boundary; 881 882 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 883 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 884 cmd.data2 = len * 0x10000; 885 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 886 if (status != 0) { 887 test = "read"; 888 goto abort; 889 } 890 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 891 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 892 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 893 cmd.data2 = len * 0x1; 894 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 895 if (status != 0) { 896 test = "write"; 897 goto abort; 898 } 899 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 900 901 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 902 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 903 cmd.data2 = len * 0x10001; 904 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 905 if (status != 0) { 906 test = "read/write"; 907 goto abort; 908 } 909 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 910 (cmd.data0 & 0xffff); 911 912abort: 913 dma_unmap_page(&mgp->pdev->dev, dmatest_bus, PAGE_SIZE, 914 DMA_BIDIRECTIONAL); 915 put_page(dmatest_page); 916 917 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 918 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 919 test, status); 920 921 return status; 922} 923 924static int myri10ge_reset(struct myri10ge_priv *mgp) 925{ 926 struct myri10ge_cmd cmd; 927 struct myri10ge_slice_state *ss; 928 int i, status; 929 size_t bytes; 930#ifdef CONFIG_MYRI10GE_DCA 931 unsigned long dca_tag_off; 932#endif 933 934 /* try to send a reset command to the card to see if it 935 * is alive */ 936 memset(&cmd, 0, sizeof(cmd)); 937 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 938 if (status != 0) { 939 dev_err(&mgp->pdev->dev, "failed reset\n"); 940 return -ENXIO; 941 } 942 943 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 944 /* 945 * Use non-ndis mcp_slot (eg, 4 bytes total, 946 * no toeplitz hash value returned. Older firmware will 947 * not understand this command, but will use the correct 948 * sized mcp_slot, so we ignore error returns 949 */ 950 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 951 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 952 953 /* Now exchange information about interrupts */ 954 955 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 956 cmd.data0 = (u32) bytes; 957 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 958 959 /* 960 * Even though we already know how many slices are supported 961 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 962 * has magic side effects, and must be called after a reset. 963 * It must be called prior to calling any RSS related cmds, 964 * including assigning an interrupt queue for anything but 965 * slice 0. It must also be called *after* 966 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 967 * the firmware to compute offsets. 968 */ 969 970 if (mgp->num_slices > 1) { 971 972 /* ask the maximum number of slices it supports */ 973 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 974 &cmd, 0); 975 if (status != 0) { 976 dev_err(&mgp->pdev->dev, 977 "failed to get number of slices\n"); 978 } 979 980 /* 981 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 982 * to setting up the interrupt queue DMA 983 */ 984 985 cmd.data0 = mgp->num_slices; 986 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 987 if (mgp->dev->real_num_tx_queues > 1) 988 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 989 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 990 &cmd, 0); 991 992 /* Firmware older than 1.4.32 only supports multiple 993 * RX queues, so if we get an error, first retry using a 994 * single TX queue before giving up */ 995 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 996 netif_set_real_num_tx_queues(mgp->dev, 1); 997 cmd.data0 = mgp->num_slices; 998 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 999 status = myri10ge_send_cmd(mgp, 1000 MXGEFW_CMD_ENABLE_RSS_QUEUES, 1001 &cmd, 0); 1002 } 1003 1004 if (status != 0) { 1005 dev_err(&mgp->pdev->dev, 1006 "failed to set number of slices\n"); 1007 1008 return status; 1009 } 1010 } 1011 for (i = 0; i < mgp->num_slices; i++) { 1012 ss = &mgp->ss[i]; 1013 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1014 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1015 cmd.data2 = i; 1016 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1017 &cmd, 0); 1018 } 1019 1020 status |= 1021 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1022 for (i = 0; i < mgp->num_slices; i++) { 1023 ss = &mgp->ss[i]; 1024 ss->irq_claim = 1025 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1026 } 1027 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1028 &cmd, 0); 1029 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1030 1031 status |= myri10ge_send_cmd 1032 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1033 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1034 if (status != 0) { 1035 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1036 return status; 1037 } 1038 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1039 1040#ifdef CONFIG_MYRI10GE_DCA 1041 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1042 dca_tag_off = cmd.data0; 1043 for (i = 0; i < mgp->num_slices; i++) { 1044 ss = &mgp->ss[i]; 1045 if (status == 0) { 1046 ss->dca_tag = (__iomem __be32 *) 1047 (mgp->sram + dca_tag_off + 4 * i); 1048 } else { 1049 ss->dca_tag = NULL; 1050 } 1051 } 1052#endif /* CONFIG_MYRI10GE_DCA */ 1053 1054 /* reset mcp/driver shared state back to 0 */ 1055 1056 mgp->link_changes = 0; 1057 for (i = 0; i < mgp->num_slices; i++) { 1058 ss = &mgp->ss[i]; 1059 1060 memset(ss->rx_done.entry, 0, bytes); 1061 ss->tx.req = 0; 1062 ss->tx.done = 0; 1063 ss->tx.pkt_start = 0; 1064 ss->tx.pkt_done = 0; 1065 ss->rx_big.cnt = 0; 1066 ss->rx_small.cnt = 0; 1067 ss->rx_done.idx = 0; 1068 ss->rx_done.cnt = 0; 1069 ss->tx.wake_queue = 0; 1070 ss->tx.stop_queue = 0; 1071 } 1072 1073 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1074 myri10ge_change_pause(mgp, mgp->pause); 1075 myri10ge_set_multicast_list(mgp->dev); 1076 return status; 1077} 1078 1079#ifdef CONFIG_MYRI10GE_DCA 1080static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1081{ 1082 int ret; 1083 u16 ctl; 1084 1085 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1086 1087 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1088 if (ret != on) { 1089 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1090 ctl |= (on << 4); 1091 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1092 } 1093 return ret; 1094} 1095 1096static void 1097myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1098{ 1099 ss->cached_dca_tag = tag; 1100 put_be32(htonl(tag), ss->dca_tag); 1101} 1102 1103static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1104{ 1105 int cpu = get_cpu(); 1106 int tag; 1107 1108 if (cpu != ss->cpu) { 1109 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1110 if (ss->cached_dca_tag != tag) 1111 myri10ge_write_dca(ss, cpu, tag); 1112 ss->cpu = cpu; 1113 } 1114 put_cpu(); 1115} 1116 1117static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1118{ 1119 int err, i; 1120 struct pci_dev *pdev = mgp->pdev; 1121 1122 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1123 return; 1124 if (!myri10ge_dca) { 1125 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1126 return; 1127 } 1128 err = dca_add_requester(&pdev->dev); 1129 if (err) { 1130 if (err != -ENODEV) 1131 dev_err(&pdev->dev, 1132 "dca_add_requester() failed, err=%d\n", err); 1133 return; 1134 } 1135 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1136 mgp->dca_enabled = 1; 1137 for (i = 0; i < mgp->num_slices; i++) { 1138 mgp->ss[i].cpu = -1; 1139 mgp->ss[i].cached_dca_tag = -1; 1140 myri10ge_update_dca(&mgp->ss[i]); 1141 } 1142} 1143 1144static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1145{ 1146 struct pci_dev *pdev = mgp->pdev; 1147 1148 if (!mgp->dca_enabled) 1149 return; 1150 mgp->dca_enabled = 0; 1151 if (mgp->relaxed_order) 1152 myri10ge_toggle_relaxed(pdev, 1); 1153 dca_remove_requester(&pdev->dev); 1154} 1155 1156static int myri10ge_notify_dca_device(struct device *dev, void *data) 1157{ 1158 struct myri10ge_priv *mgp; 1159 unsigned long event; 1160 1161 mgp = dev_get_drvdata(dev); 1162 event = *(unsigned long *)data; 1163 1164 if (event == DCA_PROVIDER_ADD) 1165 myri10ge_setup_dca(mgp); 1166 else if (event == DCA_PROVIDER_REMOVE) 1167 myri10ge_teardown_dca(mgp); 1168 return 0; 1169} 1170#endif /* CONFIG_MYRI10GE_DCA */ 1171 1172static inline void 1173myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1174 struct mcp_kreq_ether_recv *src) 1175{ 1176 __be32 low; 1177 1178 low = src->addr_low; 1179 src->addr_low = htonl(DMA_BIT_MASK(32)); 1180 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1181 mb(); 1182 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1183 mb(); 1184 src->addr_low = low; 1185 put_be32(low, &dst->addr_low); 1186 mb(); 1187} 1188 1189static void 1190myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1191 int bytes, int watchdog) 1192{ 1193 struct page *page; 1194 dma_addr_t bus; 1195 int idx; 1196#if MYRI10GE_ALLOC_SIZE > 4096 1197 int end_offset; 1198#endif 1199 1200 if (unlikely(rx->watchdog_needed && !watchdog)) 1201 return; 1202 1203 /* try to refill entire ring */ 1204 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1205 idx = rx->fill_cnt & rx->mask; 1206 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1207 /* we can use part of previous page */ 1208 get_page(rx->page); 1209 } else { 1210 /* we need a new page */ 1211 page = 1212 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1213 MYRI10GE_ALLOC_ORDER); 1214 if (unlikely(page == NULL)) { 1215 if (rx->fill_cnt - rx->cnt < 16) 1216 rx->watchdog_needed = 1; 1217 return; 1218 } 1219 1220 bus = dma_map_page(&mgp->pdev->dev, page, 0, 1221 MYRI10GE_ALLOC_SIZE, 1222 DMA_FROM_DEVICE); 1223 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 1224 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1225 if (rx->fill_cnt - rx->cnt < 16) 1226 rx->watchdog_needed = 1; 1227 return; 1228 } 1229 1230 rx->page = page; 1231 rx->page_offset = 0; 1232 rx->bus = bus; 1233 1234 } 1235 rx->info[idx].page = rx->page; 1236 rx->info[idx].page_offset = rx->page_offset; 1237 /* note that this is the address of the start of the 1238 * page */ 1239 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1240 rx->shadow[idx].addr_low = 1241 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1242 rx->shadow[idx].addr_high = 1243 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1244 1245 /* start next packet on a cacheline boundary */ 1246 rx->page_offset += SKB_DATA_ALIGN(bytes); 1247 1248#if MYRI10GE_ALLOC_SIZE > 4096 1249 /* don't cross a 4KB boundary */ 1250 end_offset = rx->page_offset + bytes - 1; 1251 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1252 rx->page_offset = end_offset & ~4095; 1253#endif 1254 rx->fill_cnt++; 1255 1256 /* copy 8 descriptors to the firmware at a time */ 1257 if ((idx & 7) == 7) { 1258 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1259 &rx->shadow[idx - 7]); 1260 } 1261 } 1262} 1263 1264static inline void 1265myri10ge_unmap_rx_page(struct pci_dev *pdev, 1266 struct myri10ge_rx_buffer_state *info, int bytes) 1267{ 1268 /* unmap the recvd page if we're the only or last user of it */ 1269 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1270 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1271 dma_unmap_page(&pdev->dev, (dma_unmap_addr(info, bus) 1272 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1273 MYRI10GE_ALLOC_SIZE, DMA_FROM_DEVICE); 1274 } 1275} 1276 1277/* 1278 * GRO does not support acceleration of tagged vlan frames, and 1279 * this NIC does not support vlan tag offload, so we must pop 1280 * the tag ourselves to be able to achieve GRO performance that 1281 * is comparable to LRO. 1282 */ 1283 1284static inline void 1285myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1286{ 1287 u8 *va; 1288 struct vlan_ethhdr *veh; 1289 skb_frag_t *frag; 1290 __wsum vsum; 1291 1292 va = addr; 1293 va += MXGEFW_PAD; 1294 veh = (struct vlan_ethhdr *)va; 1295 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1296 NETIF_F_HW_VLAN_CTAG_RX && 1297 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1298 /* fixup csum if needed */ 1299 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1300 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1301 skb->csum = csum_sub(skb->csum, vsum); 1302 } 1303 /* pop tag */ 1304 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1305 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1306 skb->len -= VLAN_HLEN; 1307 skb->data_len -= VLAN_HLEN; 1308 frag = skb_shinfo(skb)->frags; 1309 skb_frag_off_add(frag, VLAN_HLEN); 1310 skb_frag_size_sub(frag, VLAN_HLEN); 1311 } 1312} 1313 1314#define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1315 1316static inline int 1317myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1318{ 1319 struct myri10ge_priv *mgp = ss->mgp; 1320 struct sk_buff *skb; 1321 skb_frag_t *rx_frags; 1322 struct myri10ge_rx_buf *rx; 1323 int i, idx, remainder, bytes; 1324 struct pci_dev *pdev = mgp->pdev; 1325 struct net_device *dev = mgp->dev; 1326 u8 *va; 1327 1328 if (len <= mgp->small_bytes) { 1329 rx = &ss->rx_small; 1330 bytes = mgp->small_bytes; 1331 } else { 1332 rx = &ss->rx_big; 1333 bytes = mgp->big_bytes; 1334 } 1335 1336 len += MXGEFW_PAD; 1337 idx = rx->cnt & rx->mask; 1338 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1339 prefetch(va); 1340 1341 skb = napi_get_frags(&ss->napi); 1342 if (unlikely(skb == NULL)) { 1343 ss->stats.rx_dropped++; 1344 for (i = 0, remainder = len; remainder > 0; i++) { 1345 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1346 put_page(rx->info[idx].page); 1347 rx->cnt++; 1348 idx = rx->cnt & rx->mask; 1349 remainder -= MYRI10GE_ALLOC_SIZE; 1350 } 1351 return 0; 1352 } 1353 rx_frags = skb_shinfo(skb)->frags; 1354 /* Fill skb_frag_t(s) with data from our receive */ 1355 for (i = 0, remainder = len; remainder > 0; i++) { 1356 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1357 skb_fill_page_desc(skb, i, rx->info[idx].page, 1358 rx->info[idx].page_offset, 1359 remainder < MYRI10GE_ALLOC_SIZE ? 1360 remainder : MYRI10GE_ALLOC_SIZE); 1361 rx->cnt++; 1362 idx = rx->cnt & rx->mask; 1363 remainder -= MYRI10GE_ALLOC_SIZE; 1364 } 1365 1366 /* remove padding */ 1367 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1368 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1369 len -= MXGEFW_PAD; 1370 1371 skb->len = len; 1372 skb->data_len = len; 1373 skb->truesize += len; 1374 if (dev->features & NETIF_F_RXCSUM) { 1375 skb->ip_summed = CHECKSUM_COMPLETE; 1376 skb->csum = csum; 1377 } 1378 myri10ge_vlan_rx(mgp->dev, va, skb); 1379 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1380 1381 napi_gro_frags(&ss->napi); 1382 1383 return 1; 1384} 1385 1386static inline void 1387myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1388{ 1389 struct pci_dev *pdev = ss->mgp->pdev; 1390 struct myri10ge_tx_buf *tx = &ss->tx; 1391 struct netdev_queue *dev_queue; 1392 struct sk_buff *skb; 1393 int idx, len; 1394 1395 while (tx->pkt_done != mcp_index) { 1396 idx = tx->done & tx->mask; 1397 skb = tx->info[idx].skb; 1398 1399 /* Mark as free */ 1400 tx->info[idx].skb = NULL; 1401 if (tx->info[idx].last) { 1402 tx->pkt_done++; 1403 tx->info[idx].last = 0; 1404 } 1405 tx->done++; 1406 len = dma_unmap_len(&tx->info[idx], len); 1407 dma_unmap_len_set(&tx->info[idx], len, 0); 1408 if (skb) { 1409 ss->stats.tx_bytes += skb->len; 1410 ss->stats.tx_packets++; 1411 dev_consume_skb_irq(skb); 1412 if (len) 1413 dma_unmap_single(&pdev->dev, 1414 dma_unmap_addr(&tx->info[idx], 1415 bus), len, 1416 DMA_TO_DEVICE); 1417 } else { 1418 if (len) 1419 dma_unmap_page(&pdev->dev, 1420 dma_unmap_addr(&tx->info[idx], 1421 bus), len, 1422 DMA_TO_DEVICE); 1423 } 1424 } 1425 1426 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1427 /* 1428 * Make a minimal effort to prevent the NIC from polling an 1429 * idle tx queue. If we can't get the lock we leave the queue 1430 * active. In this case, either a thread was about to start 1431 * using the queue anyway, or we lost a race and the NIC will 1432 * waste some of its resources polling an inactive queue for a 1433 * while. 1434 */ 1435 1436 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1437 __netif_tx_trylock(dev_queue)) { 1438 if (tx->req == tx->done) { 1439 tx->queue_active = 0; 1440 put_be32(htonl(1), tx->send_stop); 1441 mb(); 1442 } 1443 __netif_tx_unlock(dev_queue); 1444 } 1445 1446 /* start the queue if we've stopped it */ 1447 if (netif_tx_queue_stopped(dev_queue) && 1448 tx->req - tx->done < (tx->mask >> 1) && 1449 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1450 tx->wake_queue++; 1451 netif_tx_wake_queue(dev_queue); 1452 } 1453} 1454 1455static inline int 1456myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1457{ 1458 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1459 struct myri10ge_priv *mgp = ss->mgp; 1460 unsigned long rx_bytes = 0; 1461 unsigned long rx_packets = 0; 1462 unsigned long rx_ok; 1463 int idx = rx_done->idx; 1464 int cnt = rx_done->cnt; 1465 int work_done = 0; 1466 u16 length; 1467 __wsum checksum; 1468 1469 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1470 length = ntohs(rx_done->entry[idx].length); 1471 rx_done->entry[idx].length = 0; 1472 checksum = csum_unfold(rx_done->entry[idx].checksum); 1473 rx_ok = myri10ge_rx_done(ss, length, checksum); 1474 rx_packets += rx_ok; 1475 rx_bytes += rx_ok * (unsigned long)length; 1476 cnt++; 1477 idx = cnt & (mgp->max_intr_slots - 1); 1478 work_done++; 1479 } 1480 rx_done->idx = idx; 1481 rx_done->cnt = cnt; 1482 ss->stats.rx_packets += rx_packets; 1483 ss->stats.rx_bytes += rx_bytes; 1484 1485 /* restock receive rings if needed */ 1486 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1487 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1488 mgp->small_bytes + MXGEFW_PAD, 0); 1489 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1490 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1491 1492 return work_done; 1493} 1494 1495static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1496{ 1497 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1498 1499 if (unlikely(stats->stats_updated)) { 1500 unsigned link_up = ntohl(stats->link_up); 1501 if (mgp->link_state != link_up) { 1502 mgp->link_state = link_up; 1503 1504 if (mgp->link_state == MXGEFW_LINK_UP) { 1505 netif_info(mgp, link, mgp->dev, "link up\n"); 1506 netif_carrier_on(mgp->dev); 1507 mgp->link_changes++; 1508 } else { 1509 netif_info(mgp, link, mgp->dev, "link %s\n", 1510 (link_up == MXGEFW_LINK_MYRINET ? 1511 "mismatch (Myrinet detected)" : 1512 "down")); 1513 netif_carrier_off(mgp->dev); 1514 mgp->link_changes++; 1515 } 1516 } 1517 if (mgp->rdma_tags_available != 1518 ntohl(stats->rdma_tags_available)) { 1519 mgp->rdma_tags_available = 1520 ntohl(stats->rdma_tags_available); 1521 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1522 mgp->rdma_tags_available); 1523 } 1524 mgp->down_cnt += stats->link_down; 1525 if (stats->link_down) 1526 wake_up(&mgp->down_wq); 1527 } 1528} 1529 1530static int myri10ge_poll(struct napi_struct *napi, int budget) 1531{ 1532 struct myri10ge_slice_state *ss = 1533 container_of(napi, struct myri10ge_slice_state, napi); 1534 int work_done; 1535 1536#ifdef CONFIG_MYRI10GE_DCA 1537 if (ss->mgp->dca_enabled) 1538 myri10ge_update_dca(ss); 1539#endif 1540 /* process as many rx events as NAPI will allow */ 1541 work_done = myri10ge_clean_rx_done(ss, budget); 1542 1543 if (work_done < budget) { 1544 napi_complete_done(napi, work_done); 1545 put_be32(htonl(3), ss->irq_claim); 1546 } 1547 return work_done; 1548} 1549 1550static irqreturn_t myri10ge_intr(int irq, void *arg) 1551{ 1552 struct myri10ge_slice_state *ss = arg; 1553 struct myri10ge_priv *mgp = ss->mgp; 1554 struct mcp_irq_data *stats = ss->fw_stats; 1555 struct myri10ge_tx_buf *tx = &ss->tx; 1556 u32 send_done_count; 1557 int i; 1558 1559 /* an interrupt on a non-zero receive-only slice is implicitly 1560 * valid since MSI-X irqs are not shared */ 1561 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1562 napi_schedule(&ss->napi); 1563 return IRQ_HANDLED; 1564 } 1565 1566 /* make sure it is our IRQ, and that the DMA has finished */ 1567 if (unlikely(!stats->valid)) 1568 return IRQ_NONE; 1569 1570 /* low bit indicates receives are present, so schedule 1571 * napi poll handler */ 1572 if (stats->valid & 1) 1573 napi_schedule(&ss->napi); 1574 1575 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1576 put_be32(0, mgp->irq_deassert); 1577 if (!myri10ge_deassert_wait) 1578 stats->valid = 0; 1579 mb(); 1580 } else 1581 stats->valid = 0; 1582 1583 /* Wait for IRQ line to go low, if using INTx */ 1584 i = 0; 1585 while (1) { 1586 i++; 1587 /* check for transmit completes and receives */ 1588 send_done_count = ntohl(stats->send_done_count); 1589 if (send_done_count != tx->pkt_done) 1590 myri10ge_tx_done(ss, (int)send_done_count); 1591 if (unlikely(i > myri10ge_max_irq_loops)) { 1592 netdev_warn(mgp->dev, "irq stuck?\n"); 1593 stats->valid = 0; 1594 schedule_work(&mgp->watchdog_work); 1595 } 1596 if (likely(stats->valid == 0)) 1597 break; 1598 cpu_relax(); 1599 barrier(); 1600 } 1601 1602 /* Only slice 0 updates stats */ 1603 if (ss == mgp->ss) 1604 myri10ge_check_statblock(mgp); 1605 1606 put_be32(htonl(3), ss->irq_claim + 1); 1607 return IRQ_HANDLED; 1608} 1609 1610static int 1611myri10ge_get_link_ksettings(struct net_device *netdev, 1612 struct ethtool_link_ksettings *cmd) 1613{ 1614 struct myri10ge_priv *mgp = netdev_priv(netdev); 1615 char *ptr; 1616 int i; 1617 1618 cmd->base.autoneg = AUTONEG_DISABLE; 1619 cmd->base.speed = SPEED_10000; 1620 cmd->base.duplex = DUPLEX_FULL; 1621 1622 /* 1623 * parse the product code to deterimine the interface type 1624 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1625 * after the 3rd dash in the driver's cached copy of the 1626 * EEPROM's product code string. 1627 */ 1628 ptr = mgp->product_code_string; 1629 if (ptr == NULL) { 1630 netdev_err(netdev, "Missing product code\n"); 1631 return 0; 1632 } 1633 for (i = 0; i < 3; i++, ptr++) { 1634 ptr = strchr(ptr, '-'); 1635 if (ptr == NULL) { 1636 netdev_err(netdev, "Invalid product code %s\n", 1637 mgp->product_code_string); 1638 return 0; 1639 } 1640 } 1641 if (*ptr == '2') 1642 ptr++; 1643 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1644 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1645 cmd->base.port = PORT_FIBRE; 1646 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1647 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1648 } else { 1649 cmd->base.port = PORT_OTHER; 1650 } 1651 1652 return 0; 1653} 1654 1655static void 1656myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1657{ 1658 struct myri10ge_priv *mgp = netdev_priv(netdev); 1659 1660 strscpy(info->driver, "myri10ge", sizeof(info->driver)); 1661 strscpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1662 strscpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1663 strscpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1664} 1665 1666static int myri10ge_get_coalesce(struct net_device *netdev, 1667 struct ethtool_coalesce *coal, 1668 struct kernel_ethtool_coalesce *kernel_coal, 1669 struct netlink_ext_ack *extack) 1670{ 1671 struct myri10ge_priv *mgp = netdev_priv(netdev); 1672 1673 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1674 return 0; 1675} 1676 1677static int myri10ge_set_coalesce(struct net_device *netdev, 1678 struct ethtool_coalesce *coal, 1679 struct kernel_ethtool_coalesce *kernel_coal, 1680 struct netlink_ext_ack *extack) 1681{ 1682 struct myri10ge_priv *mgp = netdev_priv(netdev); 1683 1684 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1685 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1686 return 0; 1687} 1688 1689static void 1690myri10ge_get_pauseparam(struct net_device *netdev, 1691 struct ethtool_pauseparam *pause) 1692{ 1693 struct myri10ge_priv *mgp = netdev_priv(netdev); 1694 1695 pause->autoneg = 0; 1696 pause->rx_pause = mgp->pause; 1697 pause->tx_pause = mgp->pause; 1698} 1699 1700static int 1701myri10ge_set_pauseparam(struct net_device *netdev, 1702 struct ethtool_pauseparam *pause) 1703{ 1704 struct myri10ge_priv *mgp = netdev_priv(netdev); 1705 1706 if (pause->tx_pause != mgp->pause) 1707 return myri10ge_change_pause(mgp, pause->tx_pause); 1708 if (pause->rx_pause != mgp->pause) 1709 return myri10ge_change_pause(mgp, pause->rx_pause); 1710 if (pause->autoneg != 0) 1711 return -EINVAL; 1712 return 0; 1713} 1714 1715static void 1716myri10ge_get_ringparam(struct net_device *netdev, 1717 struct ethtool_ringparam *ring, 1718 struct kernel_ethtool_ringparam *kernel_ring, 1719 struct netlink_ext_ack *extack) 1720{ 1721 struct myri10ge_priv *mgp = netdev_priv(netdev); 1722 1723 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1724 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1725 ring->rx_jumbo_max_pending = 0; 1726 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1727 ring->rx_mini_pending = ring->rx_mini_max_pending; 1728 ring->rx_pending = ring->rx_max_pending; 1729 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1730 ring->tx_pending = ring->tx_max_pending; 1731} 1732 1733static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1734 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1735 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1736 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1737 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1738 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1739 "tx_heartbeat_errors", "tx_window_errors", 1740 /* device-specific stats */ 1741 "tx_boundary", "irq", "MSI", "MSIX", 1742 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1743 "serial_number", "watchdog_resets", 1744#ifdef CONFIG_MYRI10GE_DCA 1745 "dca_capable_firmware", "dca_device_present", 1746#endif 1747 "link_changes", "link_up", "dropped_link_overflow", 1748 "dropped_link_error_or_filtered", 1749 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1750 "dropped_unicast_filtered", "dropped_multicast_filtered", 1751 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1752 "dropped_no_big_buffer" 1753}; 1754 1755static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1756 "----------- slice ---------", 1757 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1758 "rx_small_cnt", "rx_big_cnt", 1759 "wake_queue", "stop_queue", "tx_linearized", 1760}; 1761 1762#define MYRI10GE_NET_STATS_LEN 21 1763#define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1764#define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1765 1766static void 1767myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1768{ 1769 struct myri10ge_priv *mgp = netdev_priv(netdev); 1770 int i; 1771 1772 switch (stringset) { 1773 case ETH_SS_STATS: 1774 memcpy(data, *myri10ge_gstrings_main_stats, 1775 sizeof(myri10ge_gstrings_main_stats)); 1776 data += sizeof(myri10ge_gstrings_main_stats); 1777 for (i = 0; i < mgp->num_slices; i++) { 1778 memcpy(data, *myri10ge_gstrings_slice_stats, 1779 sizeof(myri10ge_gstrings_slice_stats)); 1780 data += sizeof(myri10ge_gstrings_slice_stats); 1781 } 1782 break; 1783 } 1784} 1785 1786static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1787{ 1788 struct myri10ge_priv *mgp = netdev_priv(netdev); 1789 1790 switch (sset) { 1791 case ETH_SS_STATS: 1792 return MYRI10GE_MAIN_STATS_LEN + 1793 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1794 default: 1795 return -EOPNOTSUPP; 1796 } 1797} 1798 1799static void 1800myri10ge_get_ethtool_stats(struct net_device *netdev, 1801 struct ethtool_stats *stats, u64 * data) 1802{ 1803 struct myri10ge_priv *mgp = netdev_priv(netdev); 1804 struct myri10ge_slice_state *ss; 1805 struct rtnl_link_stats64 link_stats; 1806 int slice; 1807 int i; 1808 1809 /* force stats update */ 1810 memset(&link_stats, 0, sizeof(link_stats)); 1811 (void)myri10ge_get_stats(netdev, &link_stats); 1812 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1813 data[i] = ((u64 *)&link_stats)[i]; 1814 1815 data[i++] = (unsigned int)mgp->tx_boundary; 1816 data[i++] = (unsigned int)mgp->pdev->irq; 1817 data[i++] = (unsigned int)mgp->msi_enabled; 1818 data[i++] = (unsigned int)mgp->msix_enabled; 1819 data[i++] = (unsigned int)mgp->read_dma; 1820 data[i++] = (unsigned int)mgp->write_dma; 1821 data[i++] = (unsigned int)mgp->read_write_dma; 1822 data[i++] = (unsigned int)mgp->serial_number; 1823 data[i++] = (unsigned int)mgp->watchdog_resets; 1824#ifdef CONFIG_MYRI10GE_DCA 1825 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1826 data[i++] = (unsigned int)(mgp->dca_enabled); 1827#endif 1828 data[i++] = (unsigned int)mgp->link_changes; 1829 1830 /* firmware stats are useful only in the first slice */ 1831 ss = &mgp->ss[0]; 1832 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1833 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1834 data[i++] = 1835 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1836 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1837 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1838 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1839 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1840 data[i++] = 1841 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1842 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1843 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1844 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1845 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1846 1847 for (slice = 0; slice < mgp->num_slices; slice++) { 1848 ss = &mgp->ss[slice]; 1849 data[i++] = slice; 1850 data[i++] = (unsigned int)ss->tx.pkt_start; 1851 data[i++] = (unsigned int)ss->tx.pkt_done; 1852 data[i++] = (unsigned int)ss->tx.req; 1853 data[i++] = (unsigned int)ss->tx.done; 1854 data[i++] = (unsigned int)ss->rx_small.cnt; 1855 data[i++] = (unsigned int)ss->rx_big.cnt; 1856 data[i++] = (unsigned int)ss->tx.wake_queue; 1857 data[i++] = (unsigned int)ss->tx.stop_queue; 1858 data[i++] = (unsigned int)ss->tx.linearized; 1859 } 1860} 1861 1862static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1863{ 1864 struct myri10ge_priv *mgp = netdev_priv(netdev); 1865 mgp->msg_enable = value; 1866} 1867 1868static u32 myri10ge_get_msglevel(struct net_device *netdev) 1869{ 1870 struct myri10ge_priv *mgp = netdev_priv(netdev); 1871 return mgp->msg_enable; 1872} 1873 1874/* 1875 * Use a low-level command to change the LED behavior. Rather than 1876 * blinking (which is the normal case), when identify is used, the 1877 * yellow LED turns solid. 1878 */ 1879static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1880{ 1881 struct mcp_gen_header *hdr; 1882 struct device *dev = &mgp->pdev->dev; 1883 size_t hdr_off, pattern_off, hdr_len; 1884 u32 pattern = 0xfffffffe; 1885 1886 /* find running firmware header */ 1887 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1888 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1889 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1890 (int)hdr_off); 1891 return -EIO; 1892 } 1893 hdr_len = swab32(readl(mgp->sram + hdr_off + 1894 offsetof(struct mcp_gen_header, header_length))); 1895 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1896 if (pattern_off >= (hdr_len + hdr_off)) { 1897 dev_info(dev, "Firmware does not support LED identification\n"); 1898 return -EINVAL; 1899 } 1900 if (!on) 1901 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1902 writel(swab32(pattern), mgp->sram + pattern_off); 1903 return 0; 1904} 1905 1906static int 1907myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1908{ 1909 struct myri10ge_priv *mgp = netdev_priv(netdev); 1910 int rc; 1911 1912 switch (state) { 1913 case ETHTOOL_ID_ACTIVE: 1914 rc = myri10ge_led(mgp, 1); 1915 break; 1916 1917 case ETHTOOL_ID_INACTIVE: 1918 rc = myri10ge_led(mgp, 0); 1919 break; 1920 1921 default: 1922 rc = -EINVAL; 1923 } 1924 1925 return rc; 1926} 1927 1928static const struct ethtool_ops myri10ge_ethtool_ops = { 1929 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1930 .get_drvinfo = myri10ge_get_drvinfo, 1931 .get_coalesce = myri10ge_get_coalesce, 1932 .set_coalesce = myri10ge_set_coalesce, 1933 .get_pauseparam = myri10ge_get_pauseparam, 1934 .set_pauseparam = myri10ge_set_pauseparam, 1935 .get_ringparam = myri10ge_get_ringparam, 1936 .get_link = ethtool_op_get_link, 1937 .get_strings = myri10ge_get_strings, 1938 .get_sset_count = myri10ge_get_sset_count, 1939 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1940 .set_msglevel = myri10ge_set_msglevel, 1941 .get_msglevel = myri10ge_get_msglevel, 1942 .set_phys_id = myri10ge_phys_id, 1943 .get_link_ksettings = myri10ge_get_link_ksettings, 1944}; 1945 1946static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1947{ 1948 struct myri10ge_priv *mgp = ss->mgp; 1949 struct myri10ge_cmd cmd; 1950 struct net_device *dev = mgp->dev; 1951 int tx_ring_size, rx_ring_size; 1952 int tx_ring_entries, rx_ring_entries; 1953 int i, slice, status; 1954 size_t bytes; 1955 1956 /* get ring sizes */ 1957 slice = ss - mgp->ss; 1958 cmd.data0 = slice; 1959 cmd.data1 = 0; 1960 cmd.data2 = 0; 1961 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1962 tx_ring_size = cmd.data0; 1963 cmd.data0 = slice; 1964 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1965 if (status != 0) 1966 return status; 1967 rx_ring_size = cmd.data0; 1968 1969 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1970 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1971 ss->tx.mask = tx_ring_entries - 1; 1972 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1973 1974 status = -ENOMEM; 1975 1976 /* allocate the host shadow rings */ 1977 1978 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1979 * sizeof(*ss->tx.req_list); 1980 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1981 if (ss->tx.req_bytes == NULL) 1982 goto abort_with_nothing; 1983 1984 /* ensure req_list entries are aligned to 8 bytes */ 1985 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1986 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1987 ss->tx.queue_active = 0; 1988 1989 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1990 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1991 if (ss->rx_small.shadow == NULL) 1992 goto abort_with_tx_req_bytes; 1993 1994 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1995 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1996 if (ss->rx_big.shadow == NULL) 1997 goto abort_with_rx_small_shadow; 1998 1999 /* allocate the host info rings */ 2000 2001 bytes = tx_ring_entries * sizeof(*ss->tx.info); 2002 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 2003 if (ss->tx.info == NULL) 2004 goto abort_with_rx_big_shadow; 2005 2006 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 2007 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 2008 if (ss->rx_small.info == NULL) 2009 goto abort_with_tx_info; 2010 2011 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 2012 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 2013 if (ss->rx_big.info == NULL) 2014 goto abort_with_rx_small_info; 2015 2016 /* Fill the receive rings */ 2017 ss->rx_big.cnt = 0; 2018 ss->rx_small.cnt = 0; 2019 ss->rx_big.fill_cnt = 0; 2020 ss->rx_small.fill_cnt = 0; 2021 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2022 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2023 ss->rx_small.watchdog_needed = 0; 2024 ss->rx_big.watchdog_needed = 0; 2025 if (mgp->small_bytes == 0) { 2026 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2027 } else { 2028 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2029 mgp->small_bytes + MXGEFW_PAD, 0); 2030 } 2031 2032 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2033 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2034 slice, ss->rx_small.fill_cnt); 2035 goto abort_with_rx_small_ring; 2036 } 2037 2038 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2039 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2040 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2041 slice, ss->rx_big.fill_cnt); 2042 goto abort_with_rx_big_ring; 2043 } 2044 2045 return 0; 2046 2047abort_with_rx_big_ring: 2048 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2049 int idx = i & ss->rx_big.mask; 2050 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2051 mgp->big_bytes); 2052 put_page(ss->rx_big.info[idx].page); 2053 } 2054 2055abort_with_rx_small_ring: 2056 if (mgp->small_bytes == 0) 2057 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2058 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2059 int idx = i & ss->rx_small.mask; 2060 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2061 mgp->small_bytes + MXGEFW_PAD); 2062 put_page(ss->rx_small.info[idx].page); 2063 } 2064 2065 kfree(ss->rx_big.info); 2066 2067abort_with_rx_small_info: 2068 kfree(ss->rx_small.info); 2069 2070abort_with_tx_info: 2071 kfree(ss->tx.info); 2072 2073abort_with_rx_big_shadow: 2074 kfree(ss->rx_big.shadow); 2075 2076abort_with_rx_small_shadow: 2077 kfree(ss->rx_small.shadow); 2078 2079abort_with_tx_req_bytes: 2080 kfree(ss->tx.req_bytes); 2081 ss->tx.req_bytes = NULL; 2082 ss->tx.req_list = NULL; 2083 2084abort_with_nothing: 2085 return status; 2086} 2087 2088static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2089{ 2090 struct myri10ge_priv *mgp = ss->mgp; 2091 struct sk_buff *skb; 2092 struct myri10ge_tx_buf *tx; 2093 int i, len, idx; 2094 2095 /* If not allocated, skip it */ 2096 if (ss->tx.req_list == NULL) 2097 return; 2098 2099 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2100 idx = i & ss->rx_big.mask; 2101 if (i == ss->rx_big.fill_cnt - 1) 2102 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2103 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2104 mgp->big_bytes); 2105 put_page(ss->rx_big.info[idx].page); 2106 } 2107 2108 if (mgp->small_bytes == 0) 2109 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2110 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2111 idx = i & ss->rx_small.mask; 2112 if (i == ss->rx_small.fill_cnt - 1) 2113 ss->rx_small.info[idx].page_offset = 2114 MYRI10GE_ALLOC_SIZE; 2115 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2116 mgp->small_bytes + MXGEFW_PAD); 2117 put_page(ss->rx_small.info[idx].page); 2118 } 2119 tx = &ss->tx; 2120 while (tx->done != tx->req) { 2121 idx = tx->done & tx->mask; 2122 skb = tx->info[idx].skb; 2123 2124 /* Mark as free */ 2125 tx->info[idx].skb = NULL; 2126 tx->done++; 2127 len = dma_unmap_len(&tx->info[idx], len); 2128 dma_unmap_len_set(&tx->info[idx], len, 0); 2129 if (skb) { 2130 ss->stats.tx_dropped++; 2131 dev_kfree_skb_any(skb); 2132 if (len) 2133 dma_unmap_single(&mgp->pdev->dev, 2134 dma_unmap_addr(&tx->info[idx], 2135 bus), len, 2136 DMA_TO_DEVICE); 2137 } else { 2138 if (len) 2139 dma_unmap_page(&mgp->pdev->dev, 2140 dma_unmap_addr(&tx->info[idx], 2141 bus), len, 2142 DMA_TO_DEVICE); 2143 } 2144 } 2145 kfree(ss->rx_big.info); 2146 2147 kfree(ss->rx_small.info); 2148 2149 kfree(ss->tx.info); 2150 2151 kfree(ss->rx_big.shadow); 2152 2153 kfree(ss->rx_small.shadow); 2154 2155 kfree(ss->tx.req_bytes); 2156 ss->tx.req_bytes = NULL; 2157 ss->tx.req_list = NULL; 2158} 2159 2160static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2161{ 2162 struct pci_dev *pdev = mgp->pdev; 2163 struct myri10ge_slice_state *ss; 2164 struct net_device *netdev = mgp->dev; 2165 int i; 2166 int status; 2167 2168 mgp->msi_enabled = 0; 2169 mgp->msix_enabled = 0; 2170 status = 0; 2171 if (myri10ge_msi) { 2172 if (mgp->num_slices > 1) { 2173 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2174 mgp->num_slices, mgp->num_slices); 2175 if (status < 0) { 2176 dev_err(&pdev->dev, 2177 "Error %d setting up MSI-X\n", status); 2178 return status; 2179 } 2180 mgp->msix_enabled = 1; 2181 } 2182 if (mgp->msix_enabled == 0) { 2183 status = pci_enable_msi(pdev); 2184 if (status != 0) { 2185 dev_err(&pdev->dev, 2186 "Error %d setting up MSI; falling back to xPIC\n", 2187 status); 2188 } else { 2189 mgp->msi_enabled = 1; 2190 } 2191 } 2192 } 2193 if (mgp->msix_enabled) { 2194 for (i = 0; i < mgp->num_slices; i++) { 2195 ss = &mgp->ss[i]; 2196 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2197 "%s:slice-%d", netdev->name, i); 2198 status = request_irq(mgp->msix_vectors[i].vector, 2199 myri10ge_intr, 0, ss->irq_desc, 2200 ss); 2201 if (status != 0) { 2202 dev_err(&pdev->dev, 2203 "slice %d failed to allocate IRQ\n", i); 2204 i--; 2205 while (i >= 0) { 2206 free_irq(mgp->msix_vectors[i].vector, 2207 &mgp->ss[i]); 2208 i--; 2209 } 2210 pci_disable_msix(pdev); 2211 return status; 2212 } 2213 } 2214 } else { 2215 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2216 mgp->dev->name, &mgp->ss[0]); 2217 if (status != 0) { 2218 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2219 if (mgp->msi_enabled) 2220 pci_disable_msi(pdev); 2221 } 2222 } 2223 return status; 2224} 2225 2226static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2227{ 2228 struct pci_dev *pdev = mgp->pdev; 2229 int i; 2230 2231 if (mgp->msix_enabled) { 2232 for (i = 0; i < mgp->num_slices; i++) 2233 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2234 } else { 2235 free_irq(pdev->irq, &mgp->ss[0]); 2236 } 2237 if (mgp->msi_enabled) 2238 pci_disable_msi(pdev); 2239 if (mgp->msix_enabled) 2240 pci_disable_msix(pdev); 2241} 2242 2243static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2244{ 2245 struct myri10ge_cmd cmd; 2246 struct myri10ge_slice_state *ss; 2247 int status; 2248 2249 ss = &mgp->ss[slice]; 2250 status = 0; 2251 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2252 cmd.data0 = slice; 2253 cmd.data1 = 0; 2254 cmd.data2 = 0; 2255 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2256 &cmd, 0); 2257 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2258 (mgp->sram + cmd.data0); 2259 } 2260 cmd.data0 = slice; 2261 cmd.data1 = 0; 2262 cmd.data2 = 0; 2263 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2264 &cmd, 0); 2265 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2266 (mgp->sram + cmd.data0); 2267 2268 cmd.data0 = slice; 2269 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2270 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2271 (mgp->sram + cmd.data0); 2272 2273 ss->tx.send_go = (__iomem __be32 *) 2274 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2275 ss->tx.send_stop = (__iomem __be32 *) 2276 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2277 return status; 2278 2279} 2280 2281static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2282{ 2283 struct myri10ge_cmd cmd; 2284 struct myri10ge_slice_state *ss; 2285 int status; 2286 2287 ss = &mgp->ss[slice]; 2288 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2289 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2290 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2291 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2292 if (status == -ENOSYS) { 2293 dma_addr_t bus = ss->fw_stats_bus; 2294 if (slice != 0) 2295 return -EINVAL; 2296 bus += offsetof(struct mcp_irq_data, send_done_count); 2297 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2298 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2299 status = myri10ge_send_cmd(mgp, 2300 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2301 &cmd, 0); 2302 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2303 mgp->fw_multicast_support = 0; 2304 } else { 2305 mgp->fw_multicast_support = 1; 2306 } 2307 return 0; 2308} 2309 2310static int myri10ge_open(struct net_device *dev) 2311{ 2312 struct myri10ge_slice_state *ss; 2313 struct myri10ge_priv *mgp = netdev_priv(dev); 2314 struct myri10ge_cmd cmd; 2315 int i, status, big_pow2, slice; 2316 u8 __iomem *itable; 2317 2318 if (mgp->running != MYRI10GE_ETH_STOPPED) 2319 return -EBUSY; 2320 2321 mgp->running = MYRI10GE_ETH_STARTING; 2322 status = myri10ge_reset(mgp); 2323 if (status != 0) { 2324 netdev_err(dev, "failed reset\n"); 2325 goto abort_with_nothing; 2326 } 2327 2328 if (mgp->num_slices > 1) { 2329 cmd.data0 = mgp->num_slices; 2330 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2331 cmd.data2 = 0; 2332 if (mgp->dev->real_num_tx_queues > 1) 2333 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2334 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2335 &cmd, 0); 2336 if (status != 0) { 2337 netdev_err(dev, "failed to set number of slices\n"); 2338 goto abort_with_nothing; 2339 } 2340 /* setup the indirection table */ 2341 cmd.data0 = mgp->num_slices; 2342 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2343 &cmd, 0); 2344 2345 status |= myri10ge_send_cmd(mgp, 2346 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2347 &cmd, 0); 2348 if (status != 0) { 2349 netdev_err(dev, "failed to setup rss tables\n"); 2350 goto abort_with_nothing; 2351 } 2352 2353 /* just enable an identity mapping */ 2354 itable = mgp->sram + cmd.data0; 2355 for (i = 0; i < mgp->num_slices; i++) 2356 __raw_writeb(i, &itable[i]); 2357 2358 cmd.data0 = 1; 2359 cmd.data1 = myri10ge_rss_hash; 2360 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2361 &cmd, 0); 2362 if (status != 0) { 2363 netdev_err(dev, "failed to enable slices\n"); 2364 goto abort_with_nothing; 2365 } 2366 } 2367 2368 status = myri10ge_request_irq(mgp); 2369 if (status != 0) 2370 goto abort_with_nothing; 2371 2372 /* decide what small buffer size to use. For good TCP rx 2373 * performance, it is important to not receive 1514 byte 2374 * frames into jumbo buffers, as it confuses the socket buffer 2375 * accounting code, leading to drops and erratic performance. 2376 */ 2377 2378 if (dev->mtu <= ETH_DATA_LEN) 2379 /* enough for a TCP header */ 2380 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2381 ? (128 - MXGEFW_PAD) 2382 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2383 else 2384 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2385 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2386 2387 /* Override the small buffer size? */ 2388 if (myri10ge_small_bytes >= 0) 2389 mgp->small_bytes = myri10ge_small_bytes; 2390 2391 /* Firmware needs the big buff size as a power of 2. Lie and 2392 * tell him the buffer is larger, because we only use 1 2393 * buffer/pkt, and the mtu will prevent overruns. 2394 */ 2395 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2396 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2397 while (!is_power_of_2(big_pow2)) 2398 big_pow2++; 2399 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2400 } else { 2401 big_pow2 = MYRI10GE_ALLOC_SIZE; 2402 mgp->big_bytes = big_pow2; 2403 } 2404 2405 /* setup the per-slice data structures */ 2406 for (slice = 0; slice < mgp->num_slices; slice++) { 2407 ss = &mgp->ss[slice]; 2408 2409 status = myri10ge_get_txrx(mgp, slice); 2410 if (status != 0) { 2411 netdev_err(dev, "failed to get ring sizes or locations\n"); 2412 goto abort_with_rings; 2413 } 2414 status = myri10ge_allocate_rings(ss); 2415 if (status != 0) 2416 goto abort_with_rings; 2417 2418 /* only firmware which supports multiple TX queues 2419 * supports setting up the tx stats on non-zero 2420 * slices */ 2421 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2422 status = myri10ge_set_stats(mgp, slice); 2423 if (status) { 2424 netdev_err(dev, "Couldn't set stats DMA\n"); 2425 goto abort_with_rings; 2426 } 2427 2428 /* must happen prior to any irq */ 2429 napi_enable(&(ss)->napi); 2430 } 2431 2432 /* now give firmware buffers sizes, and MTU */ 2433 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2434 cmd.data1 = 0; 2435 cmd.data2 = 0; 2436 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2437 cmd.data0 = mgp->small_bytes; 2438 status |= 2439 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2440 cmd.data0 = big_pow2; 2441 status |= 2442 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2443 if (status) { 2444 netdev_err(dev, "Couldn't set buffer sizes\n"); 2445 goto abort_with_rings; 2446 } 2447 2448 /* 2449 * Set Linux style TSO mode; this is needed only on newer 2450 * firmware versions. Older versions default to Linux 2451 * style TSO 2452 */ 2453 cmd.data0 = 0; 2454 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2455 if (status && status != -ENOSYS) { 2456 netdev_err(dev, "Couldn't set TSO mode\n"); 2457 goto abort_with_rings; 2458 } 2459 2460 mgp->link_state = ~0U; 2461 mgp->rdma_tags_available = 15; 2462 2463 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2464 if (status) { 2465 netdev_err(dev, "Couldn't bring up link\n"); 2466 goto abort_with_rings; 2467 } 2468 2469 mgp->running = MYRI10GE_ETH_RUNNING; 2470 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2471 add_timer(&mgp->watchdog_timer); 2472 netif_tx_wake_all_queues(dev); 2473 2474 return 0; 2475 2476abort_with_rings: 2477 while (slice) { 2478 slice--; 2479 napi_disable(&mgp->ss[slice].napi); 2480 } 2481 for (i = 0; i < mgp->num_slices; i++) 2482 myri10ge_free_rings(&mgp->ss[i]); 2483 2484 myri10ge_free_irq(mgp); 2485 2486abort_with_nothing: 2487 mgp->running = MYRI10GE_ETH_STOPPED; 2488 return -ENOMEM; 2489} 2490 2491static int myri10ge_close(struct net_device *dev) 2492{ 2493 struct myri10ge_priv *mgp = netdev_priv(dev); 2494 int status, old_down_cnt; 2495 int i; 2496 2497 if (mgp->running != MYRI10GE_ETH_RUNNING) 2498 return 0; 2499 2500 if (mgp->ss[0].tx.req_bytes == NULL) 2501 return 0; 2502 2503 timer_delete_sync(&mgp->watchdog_timer); 2504 mgp->running = MYRI10GE_ETH_STOPPING; 2505 for (i = 0; i < mgp->num_slices; i++) 2506 napi_disable(&mgp->ss[i].napi); 2507 2508 netif_carrier_off(dev); 2509 2510 netif_tx_stop_all_queues(dev); 2511 if (mgp->rebooted == 0) { 2512 struct myri10ge_cmd cmd; 2513 2514 old_down_cnt = mgp->down_cnt; 2515 mb(); 2516 cmd.data0 = 0; 2517 cmd.data1 = 0; 2518 cmd.data2 = 0; 2519 status = 2520 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2521 if (status) 2522 netdev_err(dev, "Couldn't bring down link\n"); 2523 2524 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2525 HZ); 2526 if (old_down_cnt == mgp->down_cnt) 2527 netdev_err(dev, "never got down irq\n"); 2528 } 2529 netif_tx_disable(dev); 2530 myri10ge_free_irq(mgp); 2531 for (i = 0; i < mgp->num_slices; i++) 2532 myri10ge_free_rings(&mgp->ss[i]); 2533 2534 mgp->running = MYRI10GE_ETH_STOPPED; 2535 return 0; 2536} 2537 2538/* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2539 * backwards one at a time and handle ring wraps */ 2540 2541static inline void 2542myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2543 struct mcp_kreq_ether_send *src, int cnt) 2544{ 2545 int idx, starting_slot; 2546 starting_slot = tx->req; 2547 while (cnt > 1) { 2548 cnt--; 2549 idx = (starting_slot + cnt) & tx->mask; 2550 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2551 mb(); 2552 } 2553} 2554 2555/* 2556 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2557 * at most 32 bytes at a time, so as to avoid involving the software 2558 * pio handler in the nic. We re-write the first segment's flags 2559 * to mark them valid only after writing the entire chain. 2560 */ 2561 2562static inline void 2563myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2564 int cnt) 2565{ 2566 int idx, i; 2567 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2568 struct mcp_kreq_ether_send *srcp; 2569 u8 last_flags; 2570 2571 idx = tx->req & tx->mask; 2572 2573 last_flags = src->flags; 2574 src->flags = 0; 2575 mb(); 2576 dst = dstp = &tx->lanai[idx]; 2577 srcp = src; 2578 2579 if ((idx + cnt) < tx->mask) { 2580 for (i = 0; i < (cnt - 1); i += 2) { 2581 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2582 mb(); /* force write every 32 bytes */ 2583 srcp += 2; 2584 dstp += 2; 2585 } 2586 } else { 2587 /* submit all but the first request, and ensure 2588 * that it is submitted below */ 2589 myri10ge_submit_req_backwards(tx, src, cnt); 2590 i = 0; 2591 } 2592 if (i < cnt) { 2593 /* submit the first request */ 2594 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2595 mb(); /* barrier before setting valid flag */ 2596 } 2597 2598 /* re-write the last 32-bits with the valid flags */ 2599 src->flags = last_flags; 2600 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2601 tx->req += cnt; 2602 mb(); 2603} 2604 2605static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2606 struct myri10ge_tx_buf *tx, int idx) 2607{ 2608 unsigned int len; 2609 int last_idx; 2610 2611 /* Free any DMA resources we've alloced and clear out the skb slot */ 2612 last_idx = (idx + 1) & tx->mask; 2613 idx = tx->req & tx->mask; 2614 do { 2615 len = dma_unmap_len(&tx->info[idx], len); 2616 if (len) { 2617 if (tx->info[idx].skb != NULL) 2618 dma_unmap_single(&mgp->pdev->dev, 2619 dma_unmap_addr(&tx->info[idx], 2620 bus), len, 2621 DMA_TO_DEVICE); 2622 else 2623 dma_unmap_page(&mgp->pdev->dev, 2624 dma_unmap_addr(&tx->info[idx], 2625 bus), len, 2626 DMA_TO_DEVICE); 2627 dma_unmap_len_set(&tx->info[idx], len, 0); 2628 tx->info[idx].skb = NULL; 2629 } 2630 idx = (idx + 1) & tx->mask; 2631 } while (idx != last_idx); 2632} 2633 2634/* 2635 * Transmit a packet. We need to split the packet so that a single 2636 * segment does not cross myri10ge->tx_boundary, so this makes segment 2637 * counting tricky. So rather than try to count segments up front, we 2638 * just give up if there are too few segments to hold a reasonably 2639 * fragmented packet currently available. If we run 2640 * out of segments while preparing a packet for DMA, we just linearize 2641 * it and try again. 2642 */ 2643 2644static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2645 struct net_device *dev) 2646{ 2647 struct myri10ge_priv *mgp = netdev_priv(dev); 2648 struct myri10ge_slice_state *ss; 2649 struct mcp_kreq_ether_send *req; 2650 struct myri10ge_tx_buf *tx; 2651 skb_frag_t *frag; 2652 struct netdev_queue *netdev_queue; 2653 dma_addr_t bus; 2654 u32 low; 2655 __be32 high_swapped; 2656 unsigned int len; 2657 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2658 u16 pseudo_hdr_offset, cksum_offset, queue; 2659 int cum_len, seglen, boundary, rdma_count; 2660 u8 flags, odd_flag; 2661 2662 queue = skb_get_queue_mapping(skb); 2663 ss = &mgp->ss[queue]; 2664 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2665 tx = &ss->tx; 2666 2667again: 2668 req = tx->req_list; 2669 avail = tx->mask - 1 - (tx->req - tx->done); 2670 2671 mss = 0; 2672 max_segments = MXGEFW_MAX_SEND_DESC; 2673 2674 if (skb_is_gso(skb)) { 2675 mss = skb_shinfo(skb)->gso_size; 2676 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2677 } 2678 2679 if ((unlikely(avail < max_segments))) { 2680 /* we are out of transmit resources */ 2681 tx->stop_queue++; 2682 netif_tx_stop_queue(netdev_queue); 2683 return NETDEV_TX_BUSY; 2684 } 2685 2686 /* Setup checksum offloading, if needed */ 2687 cksum_offset = 0; 2688 pseudo_hdr_offset = 0; 2689 odd_flag = 0; 2690 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2691 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2692 cksum_offset = skb_checksum_start_offset(skb); 2693 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2694 /* If the headers are excessively large, then we must 2695 * fall back to a software checksum */ 2696 if (unlikely(!mss && (cksum_offset > 255 || 2697 pseudo_hdr_offset > 127))) { 2698 if (skb_checksum_help(skb)) 2699 goto drop; 2700 cksum_offset = 0; 2701 pseudo_hdr_offset = 0; 2702 } else { 2703 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2704 flags |= MXGEFW_FLAGS_CKSUM; 2705 } 2706 } 2707 2708 cum_len = 0; 2709 2710 if (mss) { /* TSO */ 2711 /* this removes any CKSUM flag from before */ 2712 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2713 2714 /* negative cum_len signifies to the 2715 * send loop that we are still in the 2716 * header portion of the TSO packet. 2717 * TSO header can be at most 1KB long */ 2718 cum_len = -skb_tcp_all_headers(skb); 2719 2720 /* for IPv6 TSO, the checksum offset stores the 2721 * TCP header length, to save the firmware from 2722 * the need to parse the headers */ 2723 if (skb_is_gso_v6(skb)) { 2724 cksum_offset = tcp_hdrlen(skb); 2725 /* Can only handle headers <= max_tso6 long */ 2726 if (unlikely(-cum_len > mgp->max_tso6)) 2727 return myri10ge_sw_tso(skb, dev); 2728 } 2729 /* for TSO, pseudo_hdr_offset holds mss. 2730 * The firmware figures out where to put 2731 * the checksum by parsing the header. */ 2732 pseudo_hdr_offset = mss; 2733 } else 2734 /* Mark small packets, and pad out tiny packets */ 2735 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2736 flags |= MXGEFW_FLAGS_SMALL; 2737 2738 /* pad frames to at least ETH_ZLEN bytes */ 2739 if (eth_skb_pad(skb)) { 2740 /* The packet is gone, so we must 2741 * return 0 */ 2742 ss->stats.tx_dropped += 1; 2743 return NETDEV_TX_OK; 2744 } 2745 } 2746 2747 /* map the skb for DMA */ 2748 len = skb_headlen(skb); 2749 bus = dma_map_single(&mgp->pdev->dev, skb->data, len, DMA_TO_DEVICE); 2750 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) 2751 goto drop; 2752 2753 idx = tx->req & tx->mask; 2754 tx->info[idx].skb = skb; 2755 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2756 dma_unmap_len_set(&tx->info[idx], len, len); 2757 2758 frag_cnt = skb_shinfo(skb)->nr_frags; 2759 frag_idx = 0; 2760 count = 0; 2761 rdma_count = 0; 2762 2763 /* "rdma_count" is the number of RDMAs belonging to the 2764 * current packet BEFORE the current send request. For 2765 * non-TSO packets, this is equal to "count". 2766 * For TSO packets, rdma_count needs to be reset 2767 * to 0 after a segment cut. 2768 * 2769 * The rdma_count field of the send request is 2770 * the number of RDMAs of the packet starting at 2771 * that request. For TSO send requests with one ore more cuts 2772 * in the middle, this is the number of RDMAs starting 2773 * after the last cut in the request. All previous 2774 * segments before the last cut implicitly have 1 RDMA. 2775 * 2776 * Since the number of RDMAs is not known beforehand, 2777 * it must be filled-in retroactively - after each 2778 * segmentation cut or at the end of the entire packet. 2779 */ 2780 2781 while (1) { 2782 /* Break the SKB or Fragment up into pieces which 2783 * do not cross mgp->tx_boundary */ 2784 low = MYRI10GE_LOWPART_TO_U32(bus); 2785 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2786 while (len) { 2787 u8 flags_next; 2788 int cum_len_next; 2789 2790 if (unlikely(count == max_segments)) 2791 goto abort_linearize; 2792 2793 boundary = 2794 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2795 seglen = boundary - low; 2796 if (seglen > len) 2797 seglen = len; 2798 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2799 cum_len_next = cum_len + seglen; 2800 if (mss) { /* TSO */ 2801 (req - rdma_count)->rdma_count = rdma_count + 1; 2802 2803 if (likely(cum_len >= 0)) { /* payload */ 2804 int next_is_first, chop; 2805 2806 chop = (cum_len_next > mss); 2807 cum_len_next = cum_len_next % mss; 2808 next_is_first = (cum_len_next == 0); 2809 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2810 flags_next |= next_is_first * 2811 MXGEFW_FLAGS_FIRST; 2812 rdma_count |= -(chop | next_is_first); 2813 rdma_count += chop & ~next_is_first; 2814 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2815 int small; 2816 2817 rdma_count = -1; 2818 cum_len_next = 0; 2819 seglen = -cum_len; 2820 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2821 flags_next = MXGEFW_FLAGS_TSO_PLD | 2822 MXGEFW_FLAGS_FIRST | 2823 (small * MXGEFW_FLAGS_SMALL); 2824 } 2825 } 2826 req->addr_high = high_swapped; 2827 req->addr_low = htonl(low); 2828 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2829 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2830 req->rdma_count = 1; 2831 req->length = htons(seglen); 2832 req->cksum_offset = cksum_offset; 2833 req->flags = flags | ((cum_len & 1) * odd_flag); 2834 2835 low += seglen; 2836 len -= seglen; 2837 cum_len = cum_len_next; 2838 flags = flags_next; 2839 req++; 2840 count++; 2841 rdma_count++; 2842 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2843 if (unlikely(cksum_offset > seglen)) 2844 cksum_offset -= seglen; 2845 else 2846 cksum_offset = 0; 2847 } 2848 } 2849 if (frag_idx == frag_cnt) 2850 break; 2851 2852 /* map next fragment for DMA */ 2853 frag = &skb_shinfo(skb)->frags[frag_idx]; 2854 frag_idx++; 2855 len = skb_frag_size(frag); 2856 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2857 DMA_TO_DEVICE); 2858 if (unlikely(dma_mapping_error(&mgp->pdev->dev, bus))) { 2859 myri10ge_unmap_tx_dma(mgp, tx, idx); 2860 goto drop; 2861 } 2862 idx = (count + tx->req) & tx->mask; 2863 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2864 dma_unmap_len_set(&tx->info[idx], len, len); 2865 } 2866 2867 (req - rdma_count)->rdma_count = rdma_count; 2868 if (mss) 2869 do { 2870 req--; 2871 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2872 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2873 MXGEFW_FLAGS_FIRST))); 2874 idx = ((count - 1) + tx->req) & tx->mask; 2875 tx->info[idx].last = 1; 2876 myri10ge_submit_req(tx, tx->req_list, count); 2877 /* if using multiple tx queues, make sure NIC polls the 2878 * current slice */ 2879 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2880 tx->queue_active = 1; 2881 put_be32(htonl(1), tx->send_go); 2882 mb(); 2883 } 2884 tx->pkt_start++; 2885 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2886 tx->stop_queue++; 2887 netif_tx_stop_queue(netdev_queue); 2888 } 2889 return NETDEV_TX_OK; 2890 2891abort_linearize: 2892 myri10ge_unmap_tx_dma(mgp, tx, idx); 2893 2894 if (skb_is_gso(skb)) { 2895 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2896 goto drop; 2897 } 2898 2899 if (skb_linearize(skb)) 2900 goto drop; 2901 2902 tx->linearized++; 2903 goto again; 2904 2905drop: 2906 dev_kfree_skb_any(skb); 2907 ss->stats.tx_dropped += 1; 2908 return NETDEV_TX_OK; 2909 2910} 2911 2912static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2913 struct net_device *dev) 2914{ 2915 struct sk_buff *segs, *curr, *next; 2916 struct myri10ge_priv *mgp = netdev_priv(dev); 2917 struct myri10ge_slice_state *ss; 2918 netdev_tx_t status; 2919 2920 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2921 if (IS_ERR(segs)) 2922 goto drop; 2923 2924 skb_list_walk_safe(segs, curr, next) { 2925 skb_mark_not_on_list(curr); 2926 status = myri10ge_xmit(curr, dev); 2927 if (status != 0) { 2928 dev_kfree_skb_any(curr); 2929 skb_list_walk_safe(next, curr, next) { 2930 curr->next = NULL; 2931 dev_kfree_skb_any(curr); 2932 } 2933 goto drop; 2934 } 2935 } 2936 dev_kfree_skb_any(skb); 2937 return NETDEV_TX_OK; 2938 2939drop: 2940 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2941 dev_kfree_skb_any(skb); 2942 ss->stats.tx_dropped += 1; 2943 return NETDEV_TX_OK; 2944} 2945 2946static void myri10ge_get_stats(struct net_device *dev, 2947 struct rtnl_link_stats64 *stats) 2948{ 2949 const struct myri10ge_priv *mgp = netdev_priv(dev); 2950 const struct myri10ge_slice_netstats *slice_stats; 2951 int i; 2952 2953 for (i = 0; i < mgp->num_slices; i++) { 2954 slice_stats = &mgp->ss[i].stats; 2955 stats->rx_packets += slice_stats->rx_packets; 2956 stats->tx_packets += slice_stats->tx_packets; 2957 stats->rx_bytes += slice_stats->rx_bytes; 2958 stats->tx_bytes += slice_stats->tx_bytes; 2959 stats->rx_dropped += slice_stats->rx_dropped; 2960 stats->tx_dropped += slice_stats->tx_dropped; 2961 } 2962} 2963 2964static void myri10ge_set_multicast_list(struct net_device *dev) 2965{ 2966 struct myri10ge_priv *mgp = netdev_priv(dev); 2967 struct myri10ge_cmd cmd; 2968 struct netdev_hw_addr *ha; 2969 __be32 data[2] = { 0, 0 }; 2970 int err; 2971 2972 /* can be called from atomic contexts, 2973 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2974 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2975 2976 /* This firmware is known to not support multicast */ 2977 if (!mgp->fw_multicast_support) 2978 return; 2979 2980 /* Disable multicast filtering */ 2981 2982 cmd.data0 = 0; 2983 cmd.data1 = 0; 2984 cmd.data2 = 0; 2985 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2986 if (err != 0) { 2987 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2988 err); 2989 goto abort; 2990 } 2991 2992 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2993 /* request to disable multicast filtering, so quit here */ 2994 return; 2995 } 2996 2997 /* Flush the filters */ 2998 2999 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 3000 &cmd, 1); 3001 if (err != 0) { 3002 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 3003 err); 3004 goto abort; 3005 } 3006 3007 /* Walk the multicast list, and add each address */ 3008 netdev_for_each_mc_addr(ha, dev) { 3009 memcpy(data, &ha->addr, ETH_ALEN); 3010 cmd.data0 = ntohl(data[0]); 3011 cmd.data1 = ntohl(data[1]); 3012 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 3013 &cmd, 1); 3014 3015 if (err != 0) { 3016 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 3017 err, ha->addr); 3018 goto abort; 3019 } 3020 } 3021 /* Enable multicast filtering */ 3022 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 3023 if (err != 0) { 3024 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 3025 err); 3026 goto abort; 3027 } 3028 3029 return; 3030 3031abort: 3032 return; 3033} 3034 3035static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3036{ 3037 struct sockaddr *sa = addr; 3038 struct myri10ge_priv *mgp = netdev_priv(dev); 3039 int status; 3040 3041 if (!is_valid_ether_addr(sa->sa_data)) 3042 return -EADDRNOTAVAIL; 3043 3044 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3045 if (status != 0) { 3046 netdev_err(dev, "changing mac address failed with %d\n", 3047 status); 3048 return status; 3049 } 3050 3051 /* change the dev structure */ 3052 eth_hw_addr_set(dev, sa->sa_data); 3053 return 0; 3054} 3055 3056static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3057{ 3058 struct myri10ge_priv *mgp = netdev_priv(dev); 3059 3060 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3061 if (mgp->running) { 3062 /* if we change the mtu on an active device, we must 3063 * reset the device so the firmware sees the change */ 3064 myri10ge_close(dev); 3065 WRITE_ONCE(dev->mtu, new_mtu); 3066 myri10ge_open(dev); 3067 } else { 3068 WRITE_ONCE(dev->mtu, new_mtu); 3069 } 3070 return 0; 3071} 3072 3073/* 3074 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3075 * Only do it if the bridge is a root port since we don't want to disturb 3076 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3077 */ 3078 3079static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3080{ 3081 struct pci_dev *bridge = mgp->pdev->bus->self; 3082 struct device *dev = &mgp->pdev->dev; 3083 int cap; 3084 unsigned err_cap; 3085 int ret; 3086 3087 if (!myri10ge_ecrc_enable || !bridge) 3088 return; 3089 3090 /* check that the bridge is a root port */ 3091 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3092 if (myri10ge_ecrc_enable > 1) { 3093 struct pci_dev *prev_bridge, *old_bridge = bridge; 3094 3095 /* Walk the hierarchy up to the root port 3096 * where ECRC has to be enabled */ 3097 do { 3098 prev_bridge = bridge; 3099 bridge = bridge->bus->self; 3100 if (!bridge || prev_bridge == bridge) { 3101 dev_err(dev, 3102 "Failed to find root port" 3103 " to force ECRC\n"); 3104 return; 3105 } 3106 } while (pci_pcie_type(bridge) != 3107 PCI_EXP_TYPE_ROOT_PORT); 3108 3109 dev_info(dev, 3110 "Forcing ECRC on non-root port %s" 3111 " (enabling on root port %s)\n", 3112 pci_name(old_bridge), pci_name(bridge)); 3113 } else { 3114 dev_err(dev, 3115 "Not enabling ECRC on non-root port %s\n", 3116 pci_name(bridge)); 3117 return; 3118 } 3119 } 3120 3121 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3122 if (!cap) 3123 return; 3124 3125 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3126 if (ret) { 3127 dev_err(dev, "failed reading ext-conf-space of %s\n", 3128 pci_name(bridge)); 3129 dev_err(dev, "\t pci=nommconf in use? " 3130 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3131 return; 3132 } 3133 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3134 return; 3135 3136 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3137 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3138 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3139} 3140 3141/* 3142 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3143 * when the PCI-E Completion packets are aligned on an 8-byte 3144 * boundary. Some PCI-E chip sets always align Completion packets; on 3145 * the ones that do not, the alignment can be enforced by enabling 3146 * ECRC generation (if supported). 3147 * 3148 * When PCI-E Completion packets are not aligned, it is actually more 3149 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3150 * 3151 * If the driver can neither enable ECRC nor verify that it has 3152 * already been enabled, then it must use a firmware image which works 3153 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3154 * should also ensure that it never gives the device a Read-DMA which is 3155 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3156 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3157 * firmware image, and set tx_boundary to 4KB. 3158 */ 3159 3160static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3161{ 3162 struct pci_dev *pdev = mgp->pdev; 3163 struct device *dev = &pdev->dev; 3164 int status; 3165 3166 mgp->tx_boundary = 4096; 3167 /* 3168 * Verify the max read request size was set to 4KB 3169 * before trying the test with 4KB. 3170 */ 3171 status = pcie_get_readrq(pdev); 3172 if (status < 0) { 3173 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3174 goto abort; 3175 } 3176 if (status != 4096) { 3177 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3178 mgp->tx_boundary = 2048; 3179 } 3180 /* 3181 * load the optimized firmware (which assumes aligned PCIe 3182 * completions) in order to see if it works on this host. 3183 */ 3184 set_fw_name(mgp, myri10ge_fw_aligned, false); 3185 status = myri10ge_load_firmware(mgp, 1); 3186 if (status != 0) { 3187 goto abort; 3188 } 3189 3190 /* 3191 * Enable ECRC if possible 3192 */ 3193 myri10ge_enable_ecrc(mgp); 3194 3195 /* 3196 * Run a DMA test which watches for unaligned completions and 3197 * aborts on the first one seen. 3198 */ 3199 3200 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3201 if (status == 0) 3202 return; /* keep the aligned firmware */ 3203 3204 if (status != -E2BIG) 3205 dev_warn(dev, "DMA test failed: %d\n", status); 3206 if (status == -ENOSYS) 3207 dev_warn(dev, "Falling back to ethp! " 3208 "Please install up to date fw\n"); 3209abort: 3210 /* fall back to using the unaligned firmware */ 3211 mgp->tx_boundary = 2048; 3212 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3213} 3214 3215static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3216{ 3217 int overridden = 0; 3218 3219 if (myri10ge_force_firmware == 0) { 3220 int link_width; 3221 u16 lnk; 3222 3223 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3224 link_width = (lnk >> 4) & 0x3f; 3225 3226 /* Check to see if Link is less than 8 or if the 3227 * upstream bridge is known to provide aligned 3228 * completions */ 3229 if (link_width < 8) { 3230 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3231 link_width); 3232 mgp->tx_boundary = 4096; 3233 set_fw_name(mgp, myri10ge_fw_aligned, false); 3234 } else { 3235 myri10ge_firmware_probe(mgp); 3236 } 3237 } else { 3238 if (myri10ge_force_firmware == 1) { 3239 dev_info(&mgp->pdev->dev, 3240 "Assuming aligned completions (forced)\n"); 3241 mgp->tx_boundary = 4096; 3242 set_fw_name(mgp, myri10ge_fw_aligned, false); 3243 } else { 3244 dev_info(&mgp->pdev->dev, 3245 "Assuming unaligned completions (forced)\n"); 3246 mgp->tx_boundary = 2048; 3247 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3248 } 3249 } 3250 3251 kernel_param_lock(THIS_MODULE); 3252 if (myri10ge_fw_name != NULL) { 3253 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3254 if (fw_name) { 3255 overridden = 1; 3256 set_fw_name(mgp, fw_name, true); 3257 } 3258 } 3259 kernel_param_unlock(THIS_MODULE); 3260 3261 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3262 myri10ge_fw_names[mgp->board_number] != NULL && 3263 strlen(myri10ge_fw_names[mgp->board_number])) { 3264 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3265 overridden = 1; 3266 } 3267 if (overridden) 3268 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3269 mgp->fw_name); 3270} 3271 3272static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3273{ 3274 struct pci_dev *bridge = pdev->bus->self; 3275 int cap; 3276 u32 mask; 3277 3278 if (bridge == NULL) 3279 return; 3280 3281 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3282 if (cap) { 3283 /* a sram parity error can cause a surprise link 3284 * down; since we expect and can recover from sram 3285 * parity errors, mask surprise link down events */ 3286 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3287 mask |= 0x20; 3288 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3289 } 3290} 3291 3292static int __maybe_unused myri10ge_suspend(struct device *dev) 3293{ 3294 struct myri10ge_priv *mgp; 3295 struct net_device *netdev; 3296 3297 mgp = dev_get_drvdata(dev); 3298 if (mgp == NULL) 3299 return -EINVAL; 3300 netdev = mgp->dev; 3301 3302 netif_device_detach(netdev); 3303 if (netif_running(netdev)) { 3304 netdev_info(netdev, "closing\n"); 3305 rtnl_lock(); 3306 myri10ge_close(netdev); 3307 rtnl_unlock(); 3308 } 3309 myri10ge_dummy_rdma(mgp, 0); 3310 3311 return 0; 3312} 3313 3314static int __maybe_unused myri10ge_resume(struct device *dev) 3315{ 3316 struct pci_dev *pdev = to_pci_dev(dev); 3317 struct myri10ge_priv *mgp; 3318 struct net_device *netdev; 3319 int status; 3320 u16 vendor; 3321 3322 mgp = pci_get_drvdata(pdev); 3323 if (mgp == NULL) 3324 return -EINVAL; 3325 netdev = mgp->dev; 3326 msleep(5); /* give card time to respond */ 3327 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3328 if (vendor == 0xffff) { 3329 netdev_err(mgp->dev, "device disappeared!\n"); 3330 return -EIO; 3331 } 3332 3333 myri10ge_reset(mgp); 3334 myri10ge_dummy_rdma(mgp, 1); 3335 3336 if (netif_running(netdev)) { 3337 rtnl_lock(); 3338 status = myri10ge_open(netdev); 3339 rtnl_unlock(); 3340 if (status != 0) 3341 goto abort_with_enabled; 3342 3343 } 3344 netif_device_attach(netdev); 3345 3346 return 0; 3347 3348abort_with_enabled: 3349 return -EIO; 3350} 3351 3352static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3353{ 3354 struct pci_dev *pdev = mgp->pdev; 3355 int vs = mgp->vendor_specific_offset; 3356 u32 reboot; 3357 3358 /*enter read32 mode */ 3359 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3360 3361 /*read REBOOT_STATUS (0xfffffff0) */ 3362 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3363 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3364 return reboot; 3365} 3366 3367static void 3368myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3369 int *busy_slice_cnt, u32 rx_pause_cnt) 3370{ 3371 struct myri10ge_priv *mgp = ss->mgp; 3372 int slice = ss - mgp->ss; 3373 3374 if (ss->tx.req != ss->tx.done && 3375 ss->tx.done == ss->watchdog_tx_done && 3376 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3377 /* nic seems like it might be stuck.. */ 3378 if (rx_pause_cnt != mgp->watchdog_pause) { 3379 if (net_ratelimit()) 3380 netdev_warn(mgp->dev, "slice %d: TX paused, " 3381 "check link partner\n", slice); 3382 } else { 3383 netdev_warn(mgp->dev, 3384 "slice %d: TX stuck %d %d %d %d %d %d\n", 3385 slice, ss->tx.queue_active, ss->tx.req, 3386 ss->tx.done, ss->tx.pkt_start, 3387 ss->tx.pkt_done, 3388 (int)ntohl(mgp->ss[slice].fw_stats-> 3389 send_done_count)); 3390 *reset_needed = 1; 3391 ss->stuck = 1; 3392 } 3393 } 3394 if (ss->watchdog_tx_done != ss->tx.done || 3395 ss->watchdog_rx_done != ss->rx_done.cnt) { 3396 *busy_slice_cnt += 1; 3397 } 3398 ss->watchdog_tx_done = ss->tx.done; 3399 ss->watchdog_tx_req = ss->tx.req; 3400 ss->watchdog_rx_done = ss->rx_done.cnt; 3401} 3402 3403/* 3404 * This watchdog is used to check whether the board has suffered 3405 * from a parity error and needs to be recovered. 3406 */ 3407static void myri10ge_watchdog(struct work_struct *work) 3408{ 3409 struct myri10ge_priv *mgp = 3410 container_of(work, struct myri10ge_priv, watchdog_work); 3411 struct myri10ge_slice_state *ss; 3412 u32 reboot, rx_pause_cnt; 3413 int status, rebooted; 3414 int i; 3415 int reset_needed = 0; 3416 int busy_slice_cnt = 0; 3417 u16 cmd, vendor; 3418 3419 mgp->watchdog_resets++; 3420 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3421 rebooted = 0; 3422 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3423 /* Bus master DMA disabled? Check to see 3424 * if the card rebooted due to a parity error 3425 * For now, just report it */ 3426 reboot = myri10ge_read_reboot(mgp); 3427 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3428 reboot, myri10ge_reset_recover ? "" : " not"); 3429 if (myri10ge_reset_recover == 0) 3430 return; 3431 rtnl_lock(); 3432 mgp->rebooted = 1; 3433 rebooted = 1; 3434 myri10ge_close(mgp->dev); 3435 myri10ge_reset_recover--; 3436 mgp->rebooted = 0; 3437 /* 3438 * A rebooted nic will come back with config space as 3439 * it was after power was applied to PCIe bus. 3440 * Attempt to restore config space which was saved 3441 * when the driver was loaded, or the last time the 3442 * nic was resumed from power saving mode. 3443 */ 3444 pci_restore_state(mgp->pdev); 3445 } else { 3446 /* if we get back -1's from our slot, perhaps somebody 3447 * powered off our card. Don't try to reset it in 3448 * this case */ 3449 if (cmd == 0xffff) { 3450 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3451 if (vendor == 0xffff) { 3452 netdev_err(mgp->dev, "device disappeared!\n"); 3453 return; 3454 } 3455 } 3456 /* Perhaps it is a software error. See if stuck slice 3457 * has recovered, reset if not */ 3458 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3459 for (i = 0; i < mgp->num_slices; i++) { 3460 ss = mgp->ss; 3461 if (ss->stuck) { 3462 myri10ge_check_slice(ss, &reset_needed, 3463 &busy_slice_cnt, 3464 rx_pause_cnt); 3465 ss->stuck = 0; 3466 } 3467 } 3468 if (!reset_needed) { 3469 netdev_dbg(mgp->dev, "not resetting\n"); 3470 return; 3471 } 3472 3473 netdev_err(mgp->dev, "device timeout, resetting\n"); 3474 } 3475 3476 if (!rebooted) { 3477 rtnl_lock(); 3478 myri10ge_close(mgp->dev); 3479 } 3480 status = myri10ge_load_firmware(mgp, 1); 3481 if (status != 0) 3482 netdev_err(mgp->dev, "failed to load firmware\n"); 3483 else 3484 myri10ge_open(mgp->dev); 3485 rtnl_unlock(); 3486} 3487 3488/* 3489 * We use our own timer routine rather than relying upon 3490 * netdev->tx_timeout because we have a very large hardware transmit 3491 * queue. Due to the large queue, the netdev->tx_timeout function 3492 * cannot detect a NIC with a parity error in a timely fashion if the 3493 * NIC is lightly loaded. 3494 */ 3495static void myri10ge_watchdog_timer(struct timer_list *t) 3496{ 3497 struct myri10ge_priv *mgp; 3498 struct myri10ge_slice_state *ss; 3499 int i, reset_needed, busy_slice_cnt; 3500 u32 rx_pause_cnt; 3501 u16 cmd; 3502 3503 mgp = timer_container_of(mgp, t, watchdog_timer); 3504 3505 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3506 busy_slice_cnt = 0; 3507 for (i = 0, reset_needed = 0; 3508 i < mgp->num_slices && reset_needed == 0; ++i) { 3509 3510 ss = &mgp->ss[i]; 3511 if (ss->rx_small.watchdog_needed) { 3512 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3513 mgp->small_bytes + MXGEFW_PAD, 3514 1); 3515 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3516 myri10ge_fill_thresh) 3517 ss->rx_small.watchdog_needed = 0; 3518 } 3519 if (ss->rx_big.watchdog_needed) { 3520 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3521 mgp->big_bytes, 1); 3522 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3523 myri10ge_fill_thresh) 3524 ss->rx_big.watchdog_needed = 0; 3525 } 3526 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3527 rx_pause_cnt); 3528 } 3529 /* if we've sent or received no traffic, poll the NIC to 3530 * ensure it is still there. Otherwise, we risk not noticing 3531 * an error in a timely fashion */ 3532 if (busy_slice_cnt == 0) { 3533 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3534 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3535 reset_needed = 1; 3536 } 3537 } 3538 mgp->watchdog_pause = rx_pause_cnt; 3539 3540 if (reset_needed) { 3541 schedule_work(&mgp->watchdog_work); 3542 } else { 3543 /* rearm timer */ 3544 mod_timer(&mgp->watchdog_timer, 3545 jiffies + myri10ge_watchdog_timeout * HZ); 3546 } 3547} 3548 3549static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3550{ 3551 struct myri10ge_slice_state *ss; 3552 struct pci_dev *pdev = mgp->pdev; 3553 size_t bytes; 3554 int i; 3555 3556 if (mgp->ss == NULL) 3557 return; 3558 3559 for (i = 0; i < mgp->num_slices; i++) { 3560 ss = &mgp->ss[i]; 3561 if (ss->rx_done.entry != NULL) { 3562 bytes = mgp->max_intr_slots * 3563 sizeof(*ss->rx_done.entry); 3564 dma_free_coherent(&pdev->dev, bytes, 3565 ss->rx_done.entry, ss->rx_done.bus); 3566 ss->rx_done.entry = NULL; 3567 } 3568 if (ss->fw_stats != NULL) { 3569 bytes = sizeof(*ss->fw_stats); 3570 dma_free_coherent(&pdev->dev, bytes, 3571 ss->fw_stats, ss->fw_stats_bus); 3572 ss->fw_stats = NULL; 3573 } 3574 __netif_napi_del(&ss->napi); 3575 } 3576 /* Wait till napi structs are no longer used, and then free ss. */ 3577 synchronize_net(); 3578 kfree(mgp->ss); 3579 mgp->ss = NULL; 3580} 3581 3582static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3583{ 3584 struct myri10ge_slice_state *ss; 3585 struct pci_dev *pdev = mgp->pdev; 3586 size_t bytes; 3587 int i; 3588 3589 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3590 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3591 if (mgp->ss == NULL) { 3592 return -ENOMEM; 3593 } 3594 3595 for (i = 0; i < mgp->num_slices; i++) { 3596 ss = &mgp->ss[i]; 3597 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3598 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3599 &ss->rx_done.bus, 3600 GFP_KERNEL); 3601 if (ss->rx_done.entry == NULL) 3602 goto abort; 3603 bytes = sizeof(*ss->fw_stats); 3604 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3605 &ss->fw_stats_bus, 3606 GFP_KERNEL); 3607 if (ss->fw_stats == NULL) 3608 goto abort; 3609 ss->mgp = mgp; 3610 ss->dev = mgp->dev; 3611 netif_napi_add_weight(ss->dev, &ss->napi, myri10ge_poll, 3612 myri10ge_napi_weight); 3613 } 3614 return 0; 3615abort: 3616 myri10ge_free_slices(mgp); 3617 return -ENOMEM; 3618} 3619 3620/* 3621 * This function determines the number of slices supported. 3622 * The number slices is the minimum of the number of CPUS, 3623 * the number of MSI-X irqs supported, the number of slices 3624 * supported by the firmware 3625 */ 3626static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3627{ 3628 struct myri10ge_cmd cmd; 3629 struct pci_dev *pdev = mgp->pdev; 3630 char *old_fw; 3631 bool old_allocated; 3632 int i, status, ncpus; 3633 3634 mgp->num_slices = 1; 3635 ncpus = netif_get_num_default_rss_queues(); 3636 3637 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3638 (myri10ge_max_slices == -1 && ncpus < 2)) 3639 return; 3640 3641 /* try to load the slice aware rss firmware */ 3642 old_fw = mgp->fw_name; 3643 old_allocated = mgp->fw_name_allocated; 3644 /* don't free old_fw if we override it. */ 3645 mgp->fw_name_allocated = false; 3646 3647 if (myri10ge_fw_name != NULL) { 3648 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3649 myri10ge_fw_name); 3650 set_fw_name(mgp, myri10ge_fw_name, false); 3651 } else if (old_fw == myri10ge_fw_aligned) 3652 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3653 else 3654 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3655 status = myri10ge_load_firmware(mgp, 0); 3656 if (status != 0) { 3657 dev_info(&pdev->dev, "Rss firmware not found\n"); 3658 if (old_allocated) 3659 kfree(old_fw); 3660 return; 3661 } 3662 3663 /* hit the board with a reset to ensure it is alive */ 3664 memset(&cmd, 0, sizeof(cmd)); 3665 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3666 if (status != 0) { 3667 dev_err(&mgp->pdev->dev, "failed reset\n"); 3668 goto abort_with_fw; 3669 } 3670 3671 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3672 3673 /* tell it the size of the interrupt queues */ 3674 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3675 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3676 if (status != 0) { 3677 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3678 goto abort_with_fw; 3679 } 3680 3681 /* ask the maximum number of slices it supports */ 3682 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3683 if (status != 0) 3684 goto abort_with_fw; 3685 else 3686 mgp->num_slices = cmd.data0; 3687 3688 /* Only allow multiple slices if MSI-X is usable */ 3689 if (!myri10ge_msi) { 3690 goto abort_with_fw; 3691 } 3692 3693 /* if the admin did not specify a limit to how many 3694 * slices we should use, cap it automatically to the 3695 * number of CPUs currently online */ 3696 if (myri10ge_max_slices == -1) 3697 myri10ge_max_slices = ncpus; 3698 3699 if (mgp->num_slices > myri10ge_max_slices) 3700 mgp->num_slices = myri10ge_max_slices; 3701 3702 /* Now try to allocate as many MSI-X vectors as we have 3703 * slices. We give up on MSI-X if we can only get a single 3704 * vector. */ 3705 3706 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3707 GFP_KERNEL); 3708 if (mgp->msix_vectors == NULL) 3709 goto no_msix; 3710 for (i = 0; i < mgp->num_slices; i++) { 3711 mgp->msix_vectors[i].entry = i; 3712 } 3713 3714 while (mgp->num_slices > 1) { 3715 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3716 if (mgp->num_slices == 1) 3717 goto no_msix; 3718 status = pci_enable_msix_range(pdev, 3719 mgp->msix_vectors, 3720 mgp->num_slices, 3721 mgp->num_slices); 3722 if (status < 0) 3723 goto no_msix; 3724 3725 pci_disable_msix(pdev); 3726 3727 if (status == mgp->num_slices) { 3728 if (old_allocated) 3729 kfree(old_fw); 3730 return; 3731 } else { 3732 mgp->num_slices = status; 3733 } 3734 } 3735 3736no_msix: 3737 if (mgp->msix_vectors != NULL) { 3738 kfree(mgp->msix_vectors); 3739 mgp->msix_vectors = NULL; 3740 } 3741 3742abort_with_fw: 3743 mgp->num_slices = 1; 3744 set_fw_name(mgp, old_fw, old_allocated); 3745 myri10ge_load_firmware(mgp, 0); 3746} 3747 3748static const struct net_device_ops myri10ge_netdev_ops = { 3749 .ndo_open = myri10ge_open, 3750 .ndo_stop = myri10ge_close, 3751 .ndo_start_xmit = myri10ge_xmit, 3752 .ndo_get_stats64 = myri10ge_get_stats, 3753 .ndo_validate_addr = eth_validate_addr, 3754 .ndo_change_mtu = myri10ge_change_mtu, 3755 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3756 .ndo_set_mac_address = myri10ge_set_mac_address, 3757}; 3758 3759static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3760{ 3761 struct net_device *netdev; 3762 struct myri10ge_priv *mgp; 3763 struct device *dev = &pdev->dev; 3764 int status = -ENXIO; 3765 unsigned hdr_offset, ss_offset; 3766 static int board_number; 3767 3768 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3769 if (netdev == NULL) 3770 return -ENOMEM; 3771 3772 SET_NETDEV_DEV(netdev, &pdev->dev); 3773 3774 mgp = netdev_priv(netdev); 3775 mgp->dev = netdev; 3776 mgp->pdev = pdev; 3777 mgp->pause = myri10ge_flow_control; 3778 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3779 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3780 mgp->board_number = board_number; 3781 init_waitqueue_head(&mgp->down_wq); 3782 3783 if (pci_enable_device(pdev)) { 3784 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3785 status = -ENODEV; 3786 goto abort_with_netdev; 3787 } 3788 3789 /* Find the vendor-specific cap so we can check 3790 * the reboot register later on */ 3791 mgp->vendor_specific_offset 3792 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3793 3794 /* Set our max read request to 4KB */ 3795 status = pcie_set_readrq(pdev, 4096); 3796 if (status != 0) { 3797 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3798 status); 3799 goto abort_with_enabled; 3800 } 3801 3802 myri10ge_mask_surprise_down(pdev); 3803 pci_set_master(pdev); 3804 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3805 if (status != 0) { 3806 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3807 goto abort_with_enabled; 3808 } 3809 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3810 &mgp->cmd_bus, GFP_KERNEL); 3811 if (!mgp->cmd) { 3812 status = -ENOMEM; 3813 goto abort_with_enabled; 3814 } 3815 3816 mgp->board_span = pci_resource_len(pdev, 0); 3817 mgp->iomem_base = pci_resource_start(pdev, 0); 3818 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3819 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3820 if (mgp->sram == NULL) { 3821 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3822 mgp->board_span, mgp->iomem_base); 3823 status = -ENXIO; 3824 goto abort_with_mtrr; 3825 } 3826 hdr_offset = 3827 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3828 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3829 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3830 if (mgp->sram_size > mgp->board_span || 3831 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3832 dev_err(&pdev->dev, 3833 "invalid sram_size %dB or board span %ldB\n", 3834 mgp->sram_size, mgp->board_span); 3835 status = -EINVAL; 3836 goto abort_with_ioremap; 3837 } 3838 memcpy_fromio(mgp->eeprom_strings, 3839 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3840 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3841 status = myri10ge_read_mac_addr(mgp); 3842 if (status) 3843 goto abort_with_ioremap; 3844 3845 eth_hw_addr_set(netdev, mgp->mac_addr); 3846 3847 myri10ge_select_firmware(mgp); 3848 3849 status = myri10ge_load_firmware(mgp, 1); 3850 if (status != 0) { 3851 dev_err(&pdev->dev, "failed to load firmware\n"); 3852 goto abort_with_ioremap; 3853 } 3854 myri10ge_probe_slices(mgp); 3855 status = myri10ge_alloc_slices(mgp); 3856 if (status != 0) { 3857 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3858 goto abort_with_firmware; 3859 } 3860 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3861 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3862 status = myri10ge_reset(mgp); 3863 if (status != 0) { 3864 dev_err(&pdev->dev, "failed reset\n"); 3865 goto abort_with_slices; 3866 } 3867#ifdef CONFIG_MYRI10GE_DCA 3868 myri10ge_setup_dca(mgp); 3869#endif 3870 pci_set_drvdata(pdev, mgp); 3871 3872 /* MTU range: 68 - 9000 */ 3873 netdev->min_mtu = ETH_MIN_MTU; 3874 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3875 3876 if (myri10ge_initial_mtu > netdev->max_mtu) 3877 myri10ge_initial_mtu = netdev->max_mtu; 3878 if (myri10ge_initial_mtu < netdev->min_mtu) 3879 myri10ge_initial_mtu = netdev->min_mtu; 3880 3881 netdev->mtu = myri10ge_initial_mtu; 3882 3883 netdev->netdev_ops = &myri10ge_netdev_ops; 3884 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3885 3886 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3887 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3888 3889 netdev->features = netdev->hw_features | NETIF_F_HIGHDMA; 3890 3891 netdev->vlan_features |= mgp->features; 3892 if (mgp->fw_ver_tiny < 37) 3893 netdev->vlan_features &= ~NETIF_F_TSO6; 3894 if (mgp->fw_ver_tiny < 32) 3895 netdev->vlan_features &= ~NETIF_F_TSO; 3896 3897 /* make sure we can get an irq, and that MSI can be 3898 * setup (if available). */ 3899 status = myri10ge_request_irq(mgp); 3900 if (status != 0) 3901 goto abort_with_slices; 3902 myri10ge_free_irq(mgp); 3903 3904 /* Save configuration space to be restored if the 3905 * nic resets due to a parity error */ 3906 pci_save_state(pdev); 3907 3908 /* Setup the watchdog timer */ 3909 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3910 3911 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3912 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3913 status = register_netdev(netdev); 3914 if (status != 0) { 3915 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3916 goto abort_with_state; 3917 } 3918 if (mgp->msix_enabled) 3919 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3920 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3921 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3922 else 3923 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3924 mgp->msi_enabled ? "MSI" : "xPIC", 3925 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3926 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3927 3928 board_number++; 3929 return 0; 3930 3931abort_with_state: 3932 pci_restore_state(pdev); 3933 3934abort_with_slices: 3935 myri10ge_free_slices(mgp); 3936 3937abort_with_firmware: 3938 kfree(mgp->msix_vectors); 3939 myri10ge_dummy_rdma(mgp, 0); 3940 3941abort_with_ioremap: 3942 if (mgp->mac_addr_string != NULL) 3943 dev_err(&pdev->dev, 3944 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3945 mgp->mac_addr_string, mgp->serial_number); 3946 iounmap(mgp->sram); 3947 3948abort_with_mtrr: 3949 arch_phys_wc_del(mgp->wc_cookie); 3950 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3951 mgp->cmd, mgp->cmd_bus); 3952 3953abort_with_enabled: 3954 pci_disable_device(pdev); 3955 3956abort_with_netdev: 3957 set_fw_name(mgp, NULL, false); 3958 free_netdev(netdev); 3959 return status; 3960} 3961 3962/* 3963 * myri10ge_remove 3964 * 3965 * Does what is necessary to shutdown one Myrinet device. Called 3966 * once for each Myrinet card by the kernel when a module is 3967 * unloaded. 3968 */ 3969static void myri10ge_remove(struct pci_dev *pdev) 3970{ 3971 struct myri10ge_priv *mgp; 3972 struct net_device *netdev; 3973 3974 mgp = pci_get_drvdata(pdev); 3975 if (mgp == NULL) 3976 return; 3977 3978 cancel_work_sync(&mgp->watchdog_work); 3979 netdev = mgp->dev; 3980 unregister_netdev(netdev); 3981 3982#ifdef CONFIG_MYRI10GE_DCA 3983 myri10ge_teardown_dca(mgp); 3984#endif 3985 myri10ge_dummy_rdma(mgp, 0); 3986 3987 /* avoid a memory leak */ 3988 pci_restore_state(pdev); 3989 3990 iounmap(mgp->sram); 3991 arch_phys_wc_del(mgp->wc_cookie); 3992 myri10ge_free_slices(mgp); 3993 kfree(mgp->msix_vectors); 3994 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3995 mgp->cmd, mgp->cmd_bus); 3996 3997 set_fw_name(mgp, NULL, false); 3998 free_netdev(netdev); 3999 pci_disable_device(pdev); 4000} 4001 4002#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 4003#define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 4004 4005static const struct pci_device_id myri10ge_pci_tbl[] = { 4006 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 4007 {PCI_DEVICE 4008 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 4009 {0}, 4010}; 4011 4012MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 4013 4014static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 4015 4016static struct pci_driver myri10ge_driver = { 4017 .name = "myri10ge", 4018 .probe = myri10ge_probe, 4019 .remove = myri10ge_remove, 4020 .id_table = myri10ge_pci_tbl, 4021 .driver.pm = &myri10ge_pm_ops, 4022}; 4023 4024#ifdef CONFIG_MYRI10GE_DCA 4025static int 4026myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4027{ 4028 int err = driver_for_each_device(&myri10ge_driver.driver, 4029 NULL, &event, 4030 myri10ge_notify_dca_device); 4031 4032 if (err) 4033 return NOTIFY_BAD; 4034 return NOTIFY_DONE; 4035} 4036 4037static struct notifier_block myri10ge_dca_notifier = { 4038 .notifier_call = myri10ge_notify_dca, 4039 .next = NULL, 4040 .priority = 0, 4041}; 4042#endif /* CONFIG_MYRI10GE_DCA */ 4043 4044static __init int myri10ge_init_module(void) 4045{ 4046 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4047 4048 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4049 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4050 myri10ge_rss_hash); 4051 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4052 } 4053#ifdef CONFIG_MYRI10GE_DCA 4054 dca_register_notify(&myri10ge_dca_notifier); 4055#endif 4056 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4057 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4058 4059 return pci_register_driver(&myri10ge_driver); 4060} 4061 4062module_init(myri10ge_init_module); 4063 4064static __exit void myri10ge_cleanup_module(void) 4065{ 4066#ifdef CONFIG_MYRI10GE_DCA 4067 dca_unregister_notify(&myri10ge_dca_notifier); 4068#endif 4069 pci_unregister_driver(&myri10ge_driver); 4070} 4071 4072module_exit(myri10ge_cleanup_module);
~/
Read-only file preview.