FILE
bnxt.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/broadcom/bnxt/bnxt.c
1/* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2014-2016 Broadcom Corporation 4 * Copyright (c) 2016-2019 Broadcom Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 */ 10 11#include <linux/module.h> 12 13#include <linux/stringify.h> 14#include <linux/kernel.h> 15#include <linux/timer.h> 16#include <linux/errno.h> 17#include <linux/ioport.h> 18#include <linux/slab.h> 19#include <linux/vmalloc.h> 20#include <linux/interrupt.h> 21#include <linux/pci.h> 22#include <linux/netdevice.h> 23#include <linux/etherdevice.h> 24#include <linux/skbuff.h> 25#include <linux/dma-mapping.h> 26#include <linux/bitops.h> 27#include <linux/io.h> 28#include <linux/irq.h> 29#include <linux/delay.h> 30#include <asm/byteorder.h> 31#include <asm/page.h> 32#include <linux/time.h> 33#include <linux/mii.h> 34#include <linux/mdio.h> 35#include <linux/if.h> 36#include <linux/if_vlan.h> 37#include <linux/if_bridge.h> 38#include <linux/rtc.h> 39#include <linux/bpf.h> 40#include <net/gro.h> 41#include <net/ip.h> 42#include <net/tcp.h> 43#include <net/udp.h> 44#include <net/checksum.h> 45#include <net/ip6_checksum.h> 46#include <net/udp_tunnel.h> 47#include <linux/workqueue.h> 48#include <linux/prefetch.h> 49#include <linux/cache.h> 50#include <linux/log2.h> 51#include <linux/bitmap.h> 52#include <linux/cpu_rmap.h> 53#include <linux/cpumask.h> 54#include <net/pkt_cls.h> 55#include <net/page_pool/helpers.h> 56#include <linux/align.h> 57#include <net/netdev_lock.h> 58#include <net/netdev_queues.h> 59#include <net/netdev_rx_queue.h> 60#include <linux/pci-tph.h> 61#include <linux/bnxt/hsi.h> 62 63#include "bnxt.h" 64#include "bnxt_hwrm.h" 65#include "bnxt_ulp.h" 66#include "bnxt_sriov.h" 67#include "bnxt_ethtool.h" 68#include "bnxt_dcb.h" 69#include "bnxt_xdp.h" 70#include "bnxt_ptp.h" 71#include "bnxt_vfr.h" 72#include "bnxt_tc.h" 73#include "bnxt_devlink.h" 74#include "bnxt_debugfs.h" 75#include "bnxt_coredump.h" 76#include "bnxt_hwmon.h" 77 78#define BNXT_TX_TIMEOUT (5 * HZ) 79#define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ 80 NETIF_MSG_TX_ERR) 81 82MODULE_IMPORT_NS("NETDEV_INTERNAL"); 83MODULE_LICENSE("GPL"); 84MODULE_DESCRIPTION("Broadcom NetXtreme network driver"); 85 86#define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN) 87#define BNXT_RX_DMA_OFFSET NET_SKB_PAD 88 89#define BNXT_TX_PUSH_THRESH 164 90 91/* indexed by enum board_idx */ 92static const struct { 93 char *name; 94} board_info[] = { 95 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" }, 96 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" }, 97 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 98 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" }, 99 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" }, 100 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" }, 101 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" }, 102 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" }, 103 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" }, 104 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" }, 105 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" }, 106 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" }, 107 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" }, 108 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" }, 109 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" }, 110 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" }, 111 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" }, 112 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" }, 113 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" }, 114 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" }, 115 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" }, 116 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" }, 117 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" }, 118 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" }, 119 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" }, 120 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" }, 121 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" }, 122 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 123 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" }, 124 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 125 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 126 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" }, 127 [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" }, 128 [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" }, 129 [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" }, 130 [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" }, 131 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" }, 132 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" }, 133 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" }, 134 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" }, 135 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 136 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" }, 137 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" }, 138 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" }, 139 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" }, 140 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" }, 141 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" }, 142 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" }, 143 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" }, 144 [NETXTREME_E_P7_VF] = { "Broadcom BCM5760X Virtual Function" }, 145 [NETXTREME_E_P7_VF_HV] = { "Broadcom BCM5760X Virtual Function for Hyper-V" }, 146}; 147 148static const struct pci_device_id bnxt_pci_tbl[] = { 149 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR }, 150 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR }, 151 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 }, 152 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR }, 153 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 }, 154 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 }, 155 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 }, 156 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR }, 157 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 }, 158 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 }, 159 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 }, 160 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 }, 161 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 }, 162 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 }, 163 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR }, 164 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 }, 165 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 }, 166 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 }, 167 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 }, 168 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 }, 169 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR }, 170 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 }, 171 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP }, 172 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP }, 173 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR }, 174 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR }, 175 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP }, 176 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR }, 177 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR }, 178 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR }, 179 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR }, 180 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR }, 181 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR }, 182 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 }, 183 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 }, 184 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 }, 185 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 }, 186 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 }, 187 { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 }, 188 { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 }, 189 { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 }, 190 { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 }, 191 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR }, 192 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR }, 193 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR }, 194 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR }, 195 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR }, 196 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR }, 197 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 }, 198 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 }, 199#ifdef CONFIG_BNXT_SRIOV 200 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF }, 201 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV }, 202 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV }, 203 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF }, 204 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV }, 205 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF }, 206 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV }, 207 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV }, 208 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV }, 209 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV }, 210 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF }, 211 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF }, 212 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF }, 213 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF }, 214 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF }, 215 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV }, 216 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF }, 217 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF }, 218 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV }, 219 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV }, 220 { PCI_VDEVICE(BROADCOM, 0x1819), .driver_data = NETXTREME_E_P7_VF }, 221 { PCI_VDEVICE(BROADCOM, 0x181b), .driver_data = NETXTREME_E_P7_VF_HV }, 222 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF }, 223#endif 224 { 0 } 225}; 226 227MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl); 228 229static const u16 bnxt_vf_req_snif[] = { 230 HWRM_FUNC_CFG, 231 HWRM_FUNC_VF_CFG, 232 HWRM_PORT_PHY_QCFG, 233 HWRM_CFA_L2_FILTER_ALLOC, 234}; 235 236static const u16 bnxt_async_events_arr[] = { 237 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, 238 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, 239 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, 240 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, 241 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, 242 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, 243 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, 244 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, 245 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, 246 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, 247 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, 248 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, 249 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, 250 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, 251 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, 252 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, 253 ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER, 254}; 255 256const u16 bnxt_bstore_to_trace[] = { 257 [BNXT_CTX_SRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT_TRACE, 258 [BNXT_CTX_SRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT2_TRACE, 259 [BNXT_CTX_CRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT_TRACE, 260 [BNXT_CTX_CRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT2_TRACE, 261 [BNXT_CTX_RIGP0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP0_TRACE, 262 [BNXT_CTX_L2HWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_L2_HWRM_TRACE, 263 [BNXT_CTX_REHWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ROCE_HWRM_TRACE, 264 [BNXT_CTX_CA0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA0_TRACE, 265 [BNXT_CTX_CA1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA1_TRACE, 266 [BNXT_CTX_CA2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA2_TRACE, 267 [BNXT_CTX_RIGP1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP1_TRACE, 268 [BNXT_CTX_KONG] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_AFM_KONG_HWRM_TRACE, 269 [BNXT_CTX_QPC] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ERR_QPC_TRACE, 270}; 271 272static struct workqueue_struct *bnxt_pf_wq; 273 274#define BNXT_IPV6_MASK_ALL {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, \ 275 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }}} 276#define BNXT_IPV6_MASK_NONE {{{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}} 277 278const struct bnxt_flow_masks BNXT_FLOW_MASK_NONE = { 279 .ports = { 280 .src = 0, 281 .dst = 0, 282 }, 283 .addrs = { 284 .v6addrs = { 285 .src = BNXT_IPV6_MASK_NONE, 286 .dst = BNXT_IPV6_MASK_NONE, 287 }, 288 }, 289}; 290 291const struct bnxt_flow_masks BNXT_FLOW_IPV6_MASK_ALL = { 292 .ports = { 293 .src = cpu_to_be16(0xffff), 294 .dst = cpu_to_be16(0xffff), 295 }, 296 .addrs = { 297 .v6addrs = { 298 .src = BNXT_IPV6_MASK_ALL, 299 .dst = BNXT_IPV6_MASK_ALL, 300 }, 301 }, 302}; 303 304const struct bnxt_flow_masks BNXT_FLOW_IPV4_MASK_ALL = { 305 .ports = { 306 .src = cpu_to_be16(0xffff), 307 .dst = cpu_to_be16(0xffff), 308 }, 309 .addrs = { 310 .v4addrs = { 311 .src = cpu_to_be32(0xffffffff), 312 .dst = cpu_to_be32(0xffffffff), 313 }, 314 }, 315}; 316 317static bool bnxt_vf_pciid(enum board_idx idx) 318{ 319 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF || 320 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV || 321 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF || 322 idx == NETXTREME_E_P5_VF_HV || idx == NETXTREME_E_P7_VF || 323 idx == NETXTREME_E_P7_VF_HV); 324} 325 326#define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID) 327#define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS) 328 329#define BNXT_DB_CQ(db, idx) \ 330 writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell) 331 332#define BNXT_DB_NQ_P5(db, idx) \ 333 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\ 334 (db)->doorbell) 335 336#define BNXT_DB_NQ_P7(db, idx) \ 337 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK | \ 338 DB_RING_IDX(db, idx), (db)->doorbell) 339 340#define BNXT_DB_CQ_ARM(db, idx) \ 341 writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell) 342 343#define BNXT_DB_NQ_ARM_P5(db, idx) \ 344 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | \ 345 DB_RING_IDX(db, idx), (db)->doorbell) 346 347static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 348{ 349 if (bp->flags & BNXT_FLAG_CHIP_P7) 350 BNXT_DB_NQ_P7(db, idx); 351 else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 352 BNXT_DB_NQ_P5(db, idx); 353 else 354 BNXT_DB_CQ(db, idx); 355} 356 357static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 358{ 359 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 360 BNXT_DB_NQ_ARM_P5(db, idx); 361 else 362 BNXT_DB_CQ_ARM(db, idx); 363} 364 365static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx) 366{ 367 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 368 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL | 369 DB_RING_IDX(db, idx), db->doorbell); 370 else 371 BNXT_DB_CQ(db, idx); 372} 373 374static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay) 375{ 376 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) 377 return; 378 379 if (BNXT_PF(bp)) 380 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); 381 else 382 schedule_delayed_work(&bp->fw_reset_task, delay); 383} 384 385static void __bnxt_queue_sp_work(struct bnxt *bp) 386{ 387 if (BNXT_PF(bp)) 388 queue_work(bnxt_pf_wq, &bp->sp_task); 389 else 390 schedule_work(&bp->sp_task); 391} 392 393static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event) 394{ 395 set_bit(event, &bp->sp_event); 396 __bnxt_queue_sp_work(bp); 397} 398 399static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 400{ 401 if (!rxr->bnapi->in_reset) { 402 rxr->bnapi->in_reset = true; 403 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 404 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event); 405 else 406 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event); 407 __bnxt_queue_sp_work(bp); 408 } 409 rxr->rx_next_cons = 0xffff; 410} 411 412void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 413 u16 curr) 414{ 415 struct bnxt_napi *bnapi = txr->bnapi; 416 417 if (bnapi->tx_fault) 418 return; 419 420 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)", 421 txr->txq_index, txr->tx_hw_cons, 422 txr->tx_cons, txr->tx_prod, curr); 423 WARN_ON_ONCE(1); 424 bnapi->tx_fault = 1; 425 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 426} 427 428const u16 bnxt_lhint_arr[] = { 429 TX_BD_FLAGS_LHINT_512_AND_SMALLER, 430 TX_BD_FLAGS_LHINT_512_TO_1023, 431 TX_BD_FLAGS_LHINT_1024_TO_2047, 432 TX_BD_FLAGS_LHINT_1024_TO_2047, 433 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 434 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 435 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 436 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 437 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 438 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 439 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 440 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 441 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 442 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 443 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 444 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 445 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 446 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 447 TX_BD_FLAGS_LHINT_2048_AND_LARGER, 448}; 449 450static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) 451{ 452 struct metadata_dst *md_dst = skb_metadata_dst(skb); 453 454 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX) 455 return 0; 456 457 return md_dst->u.port_info.port_id; 458} 459 460static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 461 u16 prod) 462{ 463 /* Sync BD data before updating doorbell */ 464 wmb(); 465 bnxt_db_write(bp, &txr->tx_db, prod); 466 txr->kick_pending = 0; 467} 468 469static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) 470{ 471 struct bnxt *bp = netdev_priv(dev); 472 struct tx_bd *txbd, *txbd0; 473 struct tx_bd_ext *txbd1; 474 struct netdev_queue *txq; 475 int i; 476 dma_addr_t mapping; 477 unsigned int length, pad = 0; 478 u32 len, free_size, vlan_tag_flags, cfa_action, flags; 479 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 480 struct pci_dev *pdev = bp->pdev; 481 u16 prod, last_frag, txts_prod; 482 struct bnxt_tx_ring_info *txr; 483 struct bnxt_sw_tx_bd *tx_buf; 484 __le32 lflags = 0; 485 skb_frag_t *frag; 486 487 i = skb_get_queue_mapping(skb); 488 if (unlikely(i >= bp->tx_nr_rings)) { 489 dev_kfree_skb_any(skb); 490 dev_core_stats_tx_dropped_inc(dev); 491 return NETDEV_TX_OK; 492 } 493 494 txq = netdev_get_tx_queue(dev, i); 495 txr = &bp->tx_ring[bp->tx_ring_map[i]]; 496 prod = txr->tx_prod; 497 498#if (MAX_SKB_FRAGS > TX_MAX_FRAGS) 499 if (skb_shinfo(skb)->nr_frags > TX_MAX_FRAGS) { 500 netdev_warn_once(dev, "SKB has too many (%d) fragments, max supported is %d. SKB will be linearized.\n", 501 skb_shinfo(skb)->nr_frags, TX_MAX_FRAGS); 502 if (skb_linearize(skb)) { 503 dev_kfree_skb_any(skb); 504 dev_core_stats_tx_dropped_inc(dev); 505 return NETDEV_TX_OK; 506 } 507 } 508#endif 509 free_size = bnxt_tx_avail(bp, txr); 510 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { 511 /* We must have raced with NAPI cleanup */ 512 if (net_ratelimit() && txr->kick_pending) 513 netif_warn(bp, tx_err, dev, 514 "bnxt: ring busy w/ flush pending!\n"); 515 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 516 bp->tx_wake_thresh)) 517 return NETDEV_TX_BUSY; 518 } 519 520 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 521 goto tx_free; 522 523 length = skb->len; 524 len = skb_headlen(skb); 525 last_frag = skb_shinfo(skb)->nr_frags; 526 527 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 528 529 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 530 tx_buf->skb = skb; 531 tx_buf->nr_frags = last_frag; 532 533 vlan_tag_flags = 0; 534 cfa_action = bnxt_xmit_get_cfa_action(skb); 535 if (skb_vlan_tag_present(skb)) { 536 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | 537 skb_vlan_tag_get(skb); 538 /* Currently supports 8021Q, 8021AD vlan offloads 539 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated 540 */ 541 if (skb->vlan_proto == htons(ETH_P_8021Q)) 542 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; 543 } 544 545 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && ptp && 546 ptp->tx_tstamp_en) { 547 if (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) { 548 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 549 tx_buf->is_ts_pkt = 1; 550 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 551 } else if (!skb_is_gso(skb)) { 552 u16 seq_id, hdr_off; 553 554 if (!bnxt_ptp_parse(skb, &seq_id, &hdr_off) && 555 !bnxt_ptp_get_txts_prod(ptp, &txts_prod)) { 556 if (vlan_tag_flags) 557 hdr_off += VLAN_HLEN; 558 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP); 559 tx_buf->is_ts_pkt = 1; 560 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 561 562 ptp->txts_req[txts_prod].tx_seqid = seq_id; 563 ptp->txts_req[txts_prod].tx_hdr_off = hdr_off; 564 tx_buf->txts_prod = txts_prod; 565 } 566 } 567 } 568 if (unlikely(skb->no_fcs)) 569 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC); 570 571 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh && 572 skb_frags_readable(skb) && !lflags) { 573 struct tx_push_buffer *tx_push_buf = txr->tx_push; 574 struct tx_push_bd *tx_push = &tx_push_buf->push_bd; 575 struct tx_bd_ext *tx_push1 = &tx_push->txbd2; 576 void __iomem *db = txr->tx_db.doorbell; 577 void *pdata = tx_push_buf->data; 578 u64 *end; 579 int j, push_len; 580 581 /* Set COAL_NOW to be ready quickly for the next push */ 582 tx_push->tx_bd_len_flags_type = 583 cpu_to_le32((length << TX_BD_LEN_SHIFT) | 584 TX_BD_TYPE_LONG_TX_BD | 585 TX_BD_FLAGS_LHINT_512_AND_SMALLER | 586 TX_BD_FLAGS_COAL_NOW | 587 TX_BD_FLAGS_PACKET_END | 588 TX_BD_CNT(2)); 589 590 if (skb->ip_summed == CHECKSUM_PARTIAL) 591 tx_push1->tx_bd_hsize_lflags = 592 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 593 else 594 tx_push1->tx_bd_hsize_lflags = 0; 595 596 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 597 tx_push1->tx_bd_cfa_action = 598 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 599 600 end = pdata + length; 601 end = PTR_ALIGN(end, 8) - 1; 602 *end = 0; 603 604 skb_copy_from_linear_data(skb, pdata, len); 605 pdata += len; 606 for (j = 0; j < last_frag; j++) { 607 void *fptr; 608 609 frag = &skb_shinfo(skb)->frags[j]; 610 fptr = skb_frag_address_safe(frag); 611 if (!fptr) 612 goto normal_tx; 613 614 memcpy(pdata, fptr, skb_frag_size(frag)); 615 pdata += skb_frag_size(frag); 616 } 617 618 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type; 619 txbd->tx_bd_haddr = txr->data_mapping; 620 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2); 621 prod = NEXT_TX(prod); 622 tx_push->tx_bd_opaque = txbd->tx_bd_opaque; 623 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 624 memcpy(txbd, tx_push1, sizeof(*txbd)); 625 prod = NEXT_TX(prod); 626 tx_push->doorbell = 627 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | 628 DB_RING_IDX(&txr->tx_db, prod)); 629 WRITE_ONCE(txr->tx_prod, prod); 630 631 tx_buf->is_push = 1; 632 netdev_tx_sent_queue(txq, skb->len); 633 wmb(); /* Sync is_push and byte queue before pushing data */ 634 635 push_len = (length + sizeof(*tx_push) + 7) / 8; 636 if (push_len > 16) { 637 __iowrite64_copy(db, tx_push_buf, 16); 638 __iowrite32_copy(db + 4, tx_push_buf + 1, 639 (push_len - 16) << 1); 640 } else { 641 __iowrite64_copy(db, tx_push_buf, push_len); 642 } 643 644 goto tx_done; 645 } 646 647normal_tx: 648 if (length < BNXT_MIN_PKT_SIZE) { 649 pad = BNXT_MIN_PKT_SIZE - length; 650 if (skb_pad(skb, pad)) 651 /* SKB already freed. */ 652 goto tx_kick_pending; 653 length = BNXT_MIN_PKT_SIZE; 654 } 655 656 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); 657 658 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 659 goto tx_free; 660 661 dma_unmap_addr_set(tx_buf, mapping, mapping); 662 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | 663 TX_BD_CNT(last_frag + 2); 664 665 txbd->tx_bd_haddr = cpu_to_le64(mapping); 666 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag); 667 668 prod = NEXT_TX(prod); 669 txbd1 = (struct tx_bd_ext *) 670 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 671 672 txbd1->tx_bd_hsize_lflags = lflags; 673 if (skb_is_gso(skb)) { 674 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4); 675 u32 hdr_len; 676 677 if (skb->encapsulation) { 678 if (udp_gso) 679 hdr_len = skb_inner_transport_offset(skb) + 680 sizeof(struct udphdr); 681 else 682 hdr_len = skb_inner_tcp_all_headers(skb); 683 } else if (udp_gso) { 684 hdr_len = skb_transport_offset(skb) + 685 sizeof(struct udphdr); 686 } else { 687 hdr_len = skb_tcp_all_headers(skb); 688 } 689 690 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | 691 TX_BD_FLAGS_T_IPID | 692 (hdr_len << (TX_BD_HSIZE_SHIFT - 1))); 693 length = skb_shinfo(skb)->gso_size; 694 txbd1->tx_bd_mss = cpu_to_le32(length); 695 length += hdr_len; 696 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 697 txbd1->tx_bd_hsize_lflags |= 698 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); 699 txbd1->tx_bd_mss = 0; 700 } 701 702 length >>= 9; 703 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) { 704 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n", 705 skb->len); 706 i = 0; 707 goto tx_dma_error; 708 } 709 flags |= bnxt_lhint_arr[length]; 710 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 711 712 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); 713 txbd1->tx_bd_cfa_action = 714 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); 715 txbd0 = txbd; 716 for (i = 0; i < last_frag; i++) { 717 frag = &skb_shinfo(skb)->frags[i]; 718 prod = NEXT_TX(prod); 719 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 720 721 len = skb_frag_size(frag); 722 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len, 723 DMA_TO_DEVICE); 724 725 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) 726 goto tx_dma_error; 727 728 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 729 netmem_dma_unmap_addr_set(skb_frag_netmem(frag), tx_buf, 730 mapping, mapping); 731 732 txbd->tx_bd_haddr = cpu_to_le64(mapping); 733 734 flags = len << TX_BD_LEN_SHIFT; 735 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 736 } 737 738 flags &= ~TX_BD_LEN; 739 txbd->tx_bd_len_flags_type = 740 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | 741 TX_BD_FLAGS_PACKET_END); 742 743 netdev_tx_sent_queue(txq, skb->len); 744 745 skb_tx_timestamp(skb); 746 747 prod = NEXT_TX(prod); 748 WRITE_ONCE(txr->tx_prod, prod); 749 750 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) { 751 bnxt_txr_db_kick(bp, txr, prod); 752 } else { 753 if (free_size >= bp->tx_wake_thresh) 754 txbd0->tx_bd_len_flags_type |= 755 cpu_to_le32(TX_BD_FLAGS_NO_CMPL); 756 txr->kick_pending = 1; 757 } 758 759tx_done: 760 761 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) { 762 if (netdev_xmit_more() && !tx_buf->is_push) { 763 txbd0->tx_bd_len_flags_type &= 764 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL); 765 bnxt_txr_db_kick(bp, txr, prod); 766 } 767 768 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), 769 bp->tx_wake_thresh); 770 } 771 return NETDEV_TX_OK; 772 773tx_dma_error: 774 last_frag = i; 775 776 /* start back at beginning and unmap skb */ 777 prod = txr->tx_prod; 778 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 779 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 780 skb_headlen(skb), DMA_TO_DEVICE); 781 prod = NEXT_TX(prod); 782 783 /* unmap remaining mapped pages */ 784 for (i = 0; i < last_frag; i++) { 785 prod = NEXT_TX(prod); 786 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 787 frag = &skb_shinfo(skb)->frags[i]; 788 netmem_dma_unmap_page_attrs(&pdev->dev, 789 dma_unmap_addr(tx_buf, mapping), 790 skb_frag_size(frag), 791 DMA_TO_DEVICE, 0); 792 } 793 794tx_free: 795 dev_kfree_skb_any(skb); 796tx_kick_pending: 797 if (BNXT_TX_PTP_IS_SET(lflags)) { 798 txr->tx_buf_ring[RING_TX(bp, txr->tx_prod)].is_ts_pkt = 0; 799 atomic64_inc(&bp->ptp_cfg->stats.ts_err); 800 if (!(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP)) 801 /* set SKB to err so PTP worker will clean up */ 802 ptp->txts_req[txts_prod].tx_skb = ERR_PTR(-EIO); 803 } 804 if (txr->kick_pending) 805 bnxt_txr_db_kick(bp, txr, txr->tx_prod); 806 txr->tx_buf_ring[RING_TX(bp, txr->tx_prod)].skb = NULL; 807 dev_core_stats_tx_dropped_inc(dev); 808 return NETDEV_TX_OK; 809} 810 811/* Returns true if some remaining TX packets not processed. */ 812static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 813 int budget) 814{ 815 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 816 struct pci_dev *pdev = bp->pdev; 817 u16 hw_cons = txr->tx_hw_cons; 818 unsigned int tx_bytes = 0; 819 u16 cons = txr->tx_cons; 820 skb_frag_t *frag; 821 int tx_pkts = 0; 822 bool rc = false; 823 824 while (RING_TX(bp, cons) != hw_cons) { 825 struct bnxt_sw_tx_bd *tx_buf; 826 struct sk_buff *skb; 827 bool is_ts_pkt; 828 int j, last; 829 830 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; 831 skb = tx_buf->skb; 832 833 if (unlikely(!skb)) { 834 bnxt_sched_reset_txr(bp, txr, cons); 835 return rc; 836 } 837 838 is_ts_pkt = tx_buf->is_ts_pkt; 839 if (is_ts_pkt && (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP)) { 840 rc = true; 841 break; 842 } 843 844 cons = NEXT_TX(cons); 845 tx_pkts++; 846 tx_bytes += skb->len; 847 tx_buf->skb = NULL; 848 tx_buf->is_ts_pkt = 0; 849 850 if (tx_buf->is_push) { 851 tx_buf->is_push = 0; 852 goto next_tx_int; 853 } 854 855 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), 856 skb_headlen(skb), DMA_TO_DEVICE); 857 last = tx_buf->nr_frags; 858 859 for (j = 0; j < last; j++) { 860 frag = &skb_shinfo(skb)->frags[j]; 861 cons = NEXT_TX(cons); 862 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; 863 netmem_dma_unmap_page_attrs(&pdev->dev, 864 dma_unmap_addr(tx_buf, 865 mapping), 866 skb_frag_size(frag), 867 DMA_TO_DEVICE, 0); 868 } 869 if (unlikely(is_ts_pkt)) { 870 if (BNXT_CHIP_P5(bp)) { 871 /* PTP worker takes ownership of the skb */ 872 bnxt_get_tx_ts_p5(bp, skb, tx_buf->txts_prod); 873 skb = NULL; 874 } 875 } 876 877next_tx_int: 878 cons = NEXT_TX(cons); 879 880 napi_consume_skb(skb, budget); 881 } 882 883 WRITE_ONCE(txr->tx_cons, cons); 884 885 __netif_txq_completed_wake(txq, tx_pkts, tx_bytes, 886 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh, 887 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING); 888 889 return rc; 890} 891 892static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 893{ 894 struct bnxt_tx_ring_info *txr; 895 bool more = false; 896 int i; 897 898 bnxt_for_each_napi_tx(i, bnapi, txr) { 899 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons)) 900 more |= __bnxt_tx_int(bp, txr, budget); 901 } 902 if (!more) 903 bnapi->events &= ~BNXT_TX_CMP_EVENT; 904} 905 906static bool bnxt_separate_head_pool(struct bnxt_rx_ring_info *rxr) 907{ 908 return rxr->need_head_pool || PAGE_SIZE > BNXT_RX_PAGE_SIZE; 909} 910 911static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, 912 struct bnxt_rx_ring_info *rxr, 913 unsigned int *offset, 914 gfp_t gfp) 915{ 916 struct page *page; 917 918 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 919 page = page_pool_dev_alloc_frag(rxr->page_pool, offset, 920 BNXT_RX_PAGE_SIZE); 921 } else { 922 page = page_pool_dev_alloc_pages(rxr->page_pool); 923 *offset = 0; 924 } 925 if (!page) 926 return NULL; 927 928 *mapping = page_pool_get_dma_addr(page) + *offset; 929 return page; 930} 931 932static netmem_ref __bnxt_alloc_rx_netmem(struct bnxt *bp, dma_addr_t *mapping, 933 struct bnxt_rx_ring_info *rxr, 934 unsigned int *offset, 935 gfp_t gfp) 936{ 937 netmem_ref netmem; 938 939 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { 940 netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, BNXT_RX_PAGE_SIZE, gfp); 941 } else { 942 netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); 943 *offset = 0; 944 } 945 if (!netmem) 946 return 0; 947 948 *mapping = page_pool_get_dma_addr_netmem(netmem) + *offset; 949 return netmem; 950} 951 952static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, 953 struct bnxt_rx_ring_info *rxr, 954 gfp_t gfp) 955{ 956 unsigned int offset; 957 struct page *page; 958 959 page = page_pool_alloc_frag(rxr->head_pool, &offset, 960 bp->rx_buf_size, gfp); 961 if (!page) 962 return NULL; 963 964 *mapping = page_pool_get_dma_addr(page) + bp->rx_dma_offset + offset; 965 return page_address(page) + offset; 966} 967 968int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 969 u16 prod, gfp_t gfp) 970{ 971 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 972 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 973 dma_addr_t mapping; 974 975 if (BNXT_RX_PAGE_MODE(bp)) { 976 unsigned int offset; 977 struct page *page = 978 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp); 979 980 if (!page) 981 return -ENOMEM; 982 983 mapping += bp->rx_dma_offset; 984 rx_buf->data = page; 985 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; 986 } else { 987 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); 988 989 if (!data) 990 return -ENOMEM; 991 992 rx_buf->data = data; 993 rx_buf->data_ptr = data + bp->rx_offset; 994 } 995 rx_buf->mapping = mapping; 996 997 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 998 return 0; 999} 1000 1001void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) 1002{ 1003 u16 prod = rxr->rx_prod; 1004 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1005 struct bnxt *bp = rxr->bnapi->bp; 1006 struct rx_bd *cons_bd, *prod_bd; 1007 1008 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 1009 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1010 1011 prod_rx_buf->data = data; 1012 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; 1013 1014 prod_rx_buf->mapping = cons_rx_buf->mapping; 1015 1016 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 1017 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; 1018 1019 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr; 1020} 1021 1022static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1023{ 1024 u16 next, max = rxr->rx_agg_bmap_size; 1025 1026 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx); 1027 if (next >= max) 1028 next = find_first_zero_bit(rxr->rx_agg_bmap, max); 1029 return next; 1030} 1031 1032static int bnxt_alloc_rx_netmem(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1033 u16 prod, gfp_t gfp) 1034{ 1035 struct rx_bd *rxbd = 1036 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; 1037 struct bnxt_sw_rx_agg_bd *rx_agg_buf; 1038 u16 sw_prod = rxr->rx_sw_agg_prod; 1039 unsigned int offset = 0; 1040 dma_addr_t mapping; 1041 netmem_ref netmem; 1042 1043 netmem = __bnxt_alloc_rx_netmem(bp, &mapping, rxr, &offset, gfp); 1044 if (!netmem) 1045 return -ENOMEM; 1046 1047 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1048 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 1049 1050 __set_bit(sw_prod, rxr->rx_agg_bmap); 1051 rx_agg_buf = &rxr->rx_agg_ring[sw_prod]; 1052 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod)); 1053 1054 rx_agg_buf->netmem = netmem; 1055 rx_agg_buf->offset = offset; 1056 rx_agg_buf->mapping = mapping; 1057 rxbd->rx_bd_haddr = cpu_to_le64(mapping); 1058 rxbd->rx_bd_opaque = sw_prod; 1059 return 0; 1060} 1061 1062static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp, 1063 struct bnxt_cp_ring_info *cpr, 1064 u16 cp_cons, u16 curr) 1065{ 1066 struct rx_agg_cmp *agg; 1067 1068 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr)); 1069 agg = (struct rx_agg_cmp *) 1070 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 1071 return agg; 1072} 1073 1074static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp, 1075 struct bnxt_rx_ring_info *rxr, 1076 u16 agg_id, u16 curr) 1077{ 1078 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id]; 1079 1080 return &tpa_info->agg_arr[curr]; 1081} 1082 1083static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx, 1084 u16 start, u32 agg_bufs, bool tpa) 1085{ 1086 struct bnxt_napi *bnapi = cpr->bnapi; 1087 struct bnxt *bp = bnapi->bp; 1088 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1089 u16 prod = rxr->rx_agg_prod; 1090 u16 sw_prod = rxr->rx_sw_agg_prod; 1091 bool p5_tpa = false; 1092 u32 i; 1093 1094 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa) 1095 p5_tpa = true; 1096 1097 for (i = 0; i < agg_bufs; i++) { 1098 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf; 1099 struct rx_agg_cmp *agg; 1100 struct rx_bd *prod_bd; 1101 netmem_ref netmem; 1102 u16 cons; 1103 1104 if (p5_tpa) 1105 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i); 1106 else 1107 agg = bnxt_get_agg(bp, cpr, idx, start + i); 1108 cons = agg->rx_agg_cmp_opaque; 1109 __clear_bit(cons, rxr->rx_agg_bmap); 1110 1111 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap))) 1112 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod); 1113 1114 __set_bit(sw_prod, rxr->rx_agg_bmap); 1115 prod_rx_buf = &rxr->rx_agg_ring[sw_prod]; 1116 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1117 1118 /* It is possible for sw_prod to be equal to cons, so 1119 * set cons_rx_buf->netmem to 0 first. 1120 */ 1121 netmem = cons_rx_buf->netmem; 1122 cons_rx_buf->netmem = 0; 1123 prod_rx_buf->netmem = netmem; 1124 prod_rx_buf->offset = cons_rx_buf->offset; 1125 1126 prod_rx_buf->mapping = cons_rx_buf->mapping; 1127 1128 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)]; 1129 1130 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping); 1131 prod_bd->rx_bd_opaque = sw_prod; 1132 1133 prod = NEXT_RX_AGG(prod); 1134 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod)); 1135 } 1136 rxr->rx_agg_prod = prod; 1137 rxr->rx_sw_agg_prod = sw_prod; 1138} 1139 1140static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, 1141 struct bnxt_rx_ring_info *rxr, 1142 u16 cons, void *data, u8 *data_ptr, 1143 dma_addr_t dma_addr, 1144 unsigned int offset_and_len) 1145{ 1146 unsigned int len = offset_and_len & 0xffff; 1147 struct page *page = data; 1148 u16 prod = rxr->rx_prod; 1149 struct sk_buff *skb; 1150 int err; 1151 1152 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1153 if (unlikely(err)) { 1154 bnxt_reuse_rx_data(rxr, cons, data); 1155 return NULL; 1156 } 1157 dma_addr -= bp->rx_dma_offset; 1158 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 1159 bp->rx_dir); 1160 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE); 1161 if (!skb) { 1162 page_pool_recycle_direct(rxr->page_pool, page); 1163 return NULL; 1164 } 1165 skb_mark_for_recycle(skb); 1166 skb_reserve(skb, bp->rx_offset); 1167 __skb_put(skb, len); 1168 1169 return skb; 1170} 1171 1172static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, 1173 struct bnxt_rx_ring_info *rxr, 1174 u16 cons, void *data, u8 *data_ptr, 1175 dma_addr_t dma_addr, 1176 unsigned int offset_and_len) 1177{ 1178 unsigned int payload = offset_and_len >> 16; 1179 unsigned int len = offset_and_len & 0xffff; 1180 skb_frag_t *frag; 1181 struct page *page = data; 1182 u16 prod = rxr->rx_prod; 1183 struct sk_buff *skb; 1184 int off, err; 1185 1186 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1187 if (unlikely(err)) { 1188 bnxt_reuse_rx_data(rxr, cons, data); 1189 return NULL; 1190 } 1191 dma_addr -= bp->rx_dma_offset; 1192 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, 1193 bp->rx_dir); 1194 1195 if (unlikely(!payload)) 1196 payload = eth_get_headlen(bp->dev, data_ptr, len); 1197 1198 skb = napi_alloc_skb(&rxr->bnapi->napi, payload); 1199 if (!skb) { 1200 page_pool_recycle_direct(rxr->page_pool, page); 1201 return NULL; 1202 } 1203 1204 skb_mark_for_recycle(skb); 1205 off = (void *)data_ptr - page_address(page); 1206 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE); 1207 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, 1208 payload + NET_IP_ALIGN); 1209 1210 frag = &skb_shinfo(skb)->frags[0]; 1211 skb_frag_size_sub(frag, payload); 1212 skb_frag_off_add(frag, payload); 1213 skb->data_len -= payload; 1214 skb->tail += payload; 1215 1216 return skb; 1217} 1218 1219static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, 1220 struct bnxt_rx_ring_info *rxr, u16 cons, 1221 void *data, u8 *data_ptr, 1222 dma_addr_t dma_addr, 1223 unsigned int offset_and_len) 1224{ 1225 u16 prod = rxr->rx_prod; 1226 struct sk_buff *skb; 1227 int err; 1228 1229 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); 1230 if (unlikely(err)) { 1231 bnxt_reuse_rx_data(rxr, cons, data); 1232 return NULL; 1233 } 1234 1235 skb = napi_build_skb(data, bp->rx_buf_size); 1236 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size, 1237 bp->rx_dir); 1238 if (!skb) { 1239 page_pool_free_va(rxr->head_pool, data, true); 1240 return NULL; 1241 } 1242 1243 skb_mark_for_recycle(skb); 1244 skb_reserve(skb, bp->rx_offset); 1245 skb_put(skb, offset_and_len & 0xffff); 1246 return skb; 1247} 1248 1249static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, 1250 struct bnxt_cp_ring_info *cpr, 1251 u16 idx, u32 agg_bufs, bool tpa, 1252 struct sk_buff *skb, 1253 struct xdp_buff *xdp) 1254{ 1255 struct bnxt_napi *bnapi = cpr->bnapi; 1256 struct skb_shared_info *shinfo; 1257 struct bnxt_rx_ring_info *rxr; 1258 u32 i, total_frag_len = 0; 1259 bool p5_tpa = false; 1260 u16 prod; 1261 1262 rxr = bnapi->rx_ring; 1263 prod = rxr->rx_agg_prod; 1264 1265 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa) 1266 p5_tpa = true; 1267 1268 if (skb) 1269 shinfo = skb_shinfo(skb); 1270 else 1271 shinfo = xdp_get_shared_info_from_buff(xdp); 1272 1273 for (i = 0; i < agg_bufs; i++) { 1274 struct bnxt_sw_rx_agg_bd *cons_rx_buf; 1275 struct rx_agg_cmp *agg; 1276 u16 cons, frag_len; 1277 netmem_ref netmem; 1278 1279 if (p5_tpa) 1280 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i); 1281 else 1282 agg = bnxt_get_agg(bp, cpr, idx, i); 1283 cons = agg->rx_agg_cmp_opaque; 1284 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) & 1285 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT; 1286 1287 cons_rx_buf = &rxr->rx_agg_ring[cons]; 1288 if (skb) { 1289 skb_add_rx_frag_netmem(skb, i, cons_rx_buf->netmem, 1290 cons_rx_buf->offset, 1291 frag_len, BNXT_RX_PAGE_SIZE); 1292 } else { 1293 skb_frag_t *frag = &shinfo->frags[i]; 1294 1295 skb_frag_fill_netmem_desc(frag, cons_rx_buf->netmem, 1296 cons_rx_buf->offset, 1297 frag_len); 1298 shinfo->nr_frags = i + 1; 1299 } 1300 __clear_bit(cons, rxr->rx_agg_bmap); 1301 1302 /* It is possible for bnxt_alloc_rx_netmem() to allocate 1303 * a sw_prod index that equals the cons index, so we 1304 * need to clear the cons entry now. 1305 */ 1306 netmem = cons_rx_buf->netmem; 1307 cons_rx_buf->netmem = 0; 1308 1309 if (xdp && netmem_is_pfmemalloc(netmem)) 1310 xdp_buff_set_frag_pfmemalloc(xdp); 1311 1312 if (bnxt_alloc_rx_netmem(bp, rxr, prod, GFP_ATOMIC) != 0) { 1313 if (skb) { 1314 skb->len -= frag_len; 1315 skb->data_len -= frag_len; 1316 skb->truesize -= BNXT_RX_PAGE_SIZE; 1317 } 1318 1319 --shinfo->nr_frags; 1320 cons_rx_buf->netmem = netmem; 1321 1322 /* Update prod since possibly some netmems have been 1323 * allocated already. 1324 */ 1325 rxr->rx_agg_prod = prod; 1326 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa); 1327 return 0; 1328 } 1329 1330 page_pool_dma_sync_netmem_for_cpu(rxr->page_pool, netmem, 0, 1331 BNXT_RX_PAGE_SIZE); 1332 1333 total_frag_len += frag_len; 1334 prod = NEXT_RX_AGG(prod); 1335 } 1336 rxr->rx_agg_prod = prod; 1337 return total_frag_len; 1338} 1339 1340static struct sk_buff *bnxt_rx_agg_netmems_skb(struct bnxt *bp, 1341 struct bnxt_cp_ring_info *cpr, 1342 struct sk_buff *skb, u16 idx, 1343 u32 agg_bufs, bool tpa) 1344{ 1345 u32 total_frag_len = 0; 1346 1347 total_frag_len = __bnxt_rx_agg_netmems(bp, cpr, idx, agg_bufs, tpa, 1348 skb, NULL); 1349 if (!total_frag_len) { 1350 skb_mark_for_recycle(skb); 1351 dev_kfree_skb(skb); 1352 return NULL; 1353 } 1354 1355 return skb; 1356} 1357 1358static u32 bnxt_rx_agg_netmems_xdp(struct bnxt *bp, 1359 struct bnxt_cp_ring_info *cpr, 1360 struct xdp_buff *xdp, u16 idx, 1361 u32 agg_bufs, bool tpa) 1362{ 1363 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp); 1364 u32 total_frag_len = 0; 1365 1366 if (!xdp_buff_has_frags(xdp)) 1367 shinfo->nr_frags = 0; 1368 1369 total_frag_len = __bnxt_rx_agg_netmems(bp, cpr, idx, agg_bufs, tpa, 1370 NULL, xdp); 1371 if (total_frag_len) { 1372 xdp_buff_set_frags_flag(xdp); 1373 shinfo->nr_frags = agg_bufs; 1374 shinfo->xdp_frags_size = total_frag_len; 1375 } 1376 return total_frag_len; 1377} 1378 1379static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1380 u8 agg_bufs, u32 *raw_cons) 1381{ 1382 u16 last; 1383 struct rx_agg_cmp *agg; 1384 1385 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs); 1386 last = RING_CMP(*raw_cons); 1387 agg = (struct rx_agg_cmp *) 1388 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)]; 1389 return RX_AGG_CMP_VALID(agg, *raw_cons); 1390} 1391 1392static struct sk_buff *bnxt_copy_data(struct bnxt_napi *bnapi, u8 *data, 1393 unsigned int len, 1394 dma_addr_t mapping) 1395{ 1396 struct bnxt *bp = bnapi->bp; 1397 struct pci_dev *pdev = bp->pdev; 1398 struct sk_buff *skb; 1399 1400 skb = napi_alloc_skb(&bnapi->napi, len); 1401 if (!skb) 1402 return NULL; 1403 1404 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copybreak, 1405 bp->rx_dir); 1406 1407 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN, 1408 len + NET_IP_ALIGN); 1409 1410 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copybreak, 1411 bp->rx_dir); 1412 1413 skb_put(skb, len); 1414 1415 return skb; 1416} 1417 1418static struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data, 1419 unsigned int len, 1420 dma_addr_t mapping) 1421{ 1422 return bnxt_copy_data(bnapi, data, len, mapping); 1423} 1424 1425static struct sk_buff *bnxt_copy_xdp(struct bnxt_napi *bnapi, 1426 struct xdp_buff *xdp, 1427 unsigned int len, 1428 dma_addr_t mapping) 1429{ 1430 unsigned int metasize = 0; 1431 u8 *data = xdp->data; 1432 struct sk_buff *skb; 1433 1434 len = xdp->data_end - xdp->data_meta; 1435 metasize = xdp->data - xdp->data_meta; 1436 data = xdp->data_meta; 1437 1438 skb = bnxt_copy_data(bnapi, data, len, mapping); 1439 if (!skb) 1440 return skb; 1441 1442 if (metasize) { 1443 skb_metadata_set(skb, metasize); 1444 __skb_pull(skb, metasize); 1445 } 1446 1447 return skb; 1448} 1449 1450static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 1451 u32 *raw_cons, void *cmp) 1452{ 1453 struct rx_cmp *rxcmp = cmp; 1454 u32 tmp_raw_cons = *raw_cons; 1455 u8 cmp_type, agg_bufs = 0; 1456 1457 cmp_type = RX_CMP_TYPE(rxcmp); 1458 1459 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 1460 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & 1461 RX_CMP_AGG_BUFS) >> 1462 RX_CMP_AGG_BUFS_SHIFT; 1463 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 1464 struct rx_tpa_end_cmp *tpa_end = cmp; 1465 1466 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 1467 return 0; 1468 1469 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1470 } 1471 1472 if (agg_bufs) { 1473 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 1474 return -EBUSY; 1475 } 1476 *raw_cons = tmp_raw_cons; 1477 return 0; 1478} 1479 1480static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1481{ 1482 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1483 u16 idx = agg_id & MAX_TPA_P5_MASK; 1484 1485 if (test_bit(idx, map->agg_idx_bmap)) { 1486 idx = find_first_zero_bit(map->agg_idx_bmap, MAX_TPA_P5); 1487 if (idx >= MAX_TPA_P5) 1488 return INVALID_HW_RING_ID; 1489 } 1490 __set_bit(idx, map->agg_idx_bmap); 1491 map->agg_id_tbl[agg_id] = idx; 1492 return idx; 1493} 1494 1495static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx) 1496{ 1497 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1498 1499 __clear_bit(idx, map->agg_idx_bmap); 1500} 1501 1502static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id) 1503{ 1504 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map; 1505 1506 return map->agg_id_tbl[agg_id]; 1507} 1508 1509static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info, 1510 struct rx_tpa_start_cmp *tpa_start, 1511 struct rx_tpa_start_cmp_ext *tpa_start1) 1512{ 1513 tpa_info->cfa_code_valid = 1; 1514 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1); 1515 tpa_info->vlan_valid = 0; 1516 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) { 1517 tpa_info->vlan_valid = 1; 1518 tpa_info->metadata = 1519 le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata); 1520 } 1521} 1522 1523static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info, 1524 struct rx_tpa_start_cmp *tpa_start, 1525 struct rx_tpa_start_cmp_ext *tpa_start1) 1526{ 1527 tpa_info->vlan_valid = 0; 1528 if (TPA_START_VLAN_VALID(tpa_start)) { 1529 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start); 1530 u32 vlan_proto = ETH_P_8021Q; 1531 1532 tpa_info->vlan_valid = 1; 1533 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD) 1534 vlan_proto = ETH_P_8021AD; 1535 tpa_info->metadata = vlan_proto << 16 | 1536 TPA_START_METADATA0_TCI(tpa_start1); 1537 } 1538} 1539 1540static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1541 u8 cmp_type, struct rx_tpa_start_cmp *tpa_start, 1542 struct rx_tpa_start_cmp_ext *tpa_start1) 1543{ 1544 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf; 1545 struct bnxt_tpa_info *tpa_info; 1546 u16 cons, prod, agg_id; 1547 struct rx_bd *prod_bd; 1548 dma_addr_t mapping; 1549 1550 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 1551 agg_id = TPA_START_AGG_ID_P5(tpa_start); 1552 agg_id = bnxt_alloc_agg_idx(rxr, agg_id); 1553 if (unlikely(agg_id == INVALID_HW_RING_ID)) { 1554 netdev_warn(bp->dev, "Unable to allocate agg ID for ring %d, agg 0x%x\n", 1555 rxr->bnapi->index, 1556 TPA_START_AGG_ID_P5(tpa_start)); 1557 bnxt_sched_reset_rxr(bp, rxr); 1558 return; 1559 } 1560 } else { 1561 agg_id = TPA_START_AGG_ID(tpa_start); 1562 } 1563 cons = tpa_start->rx_tpa_start_cmp_opaque; 1564 prod = rxr->rx_prod; 1565 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1566 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; 1567 tpa_info = &rxr->rx_tpa[agg_id]; 1568 1569 if (unlikely(cons != rxr->rx_next_cons || 1570 TPA_START_ERROR(tpa_start))) { 1571 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n", 1572 cons, rxr->rx_next_cons, 1573 TPA_START_ERROR_CODE(tpa_start1)); 1574 bnxt_sched_reset_rxr(bp, rxr); 1575 return; 1576 } 1577 prod_rx_buf->data = tpa_info->data; 1578 prod_rx_buf->data_ptr = tpa_info->data_ptr; 1579 1580 mapping = tpa_info->mapping; 1581 prod_rx_buf->mapping = mapping; 1582 1583 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; 1584 1585 prod_bd->rx_bd_haddr = cpu_to_le64(mapping); 1586 1587 tpa_info->data = cons_rx_buf->data; 1588 tpa_info->data_ptr = cons_rx_buf->data_ptr; 1589 cons_rx_buf->data = NULL; 1590 tpa_info->mapping = cons_rx_buf->mapping; 1591 1592 tpa_info->len = 1593 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >> 1594 RX_TPA_START_CMP_LEN_SHIFT; 1595 if (likely(TPA_START_HASH_VALID(tpa_start))) { 1596 tpa_info->hash_type = PKT_HASH_TYPE_L4; 1597 tpa_info->gso_type = SKB_GSO_TCPV4; 1598 if (TPA_START_IS_IPV6(tpa_start1)) 1599 tpa_info->gso_type = SKB_GSO_TCPV6; 1600 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */ 1601 else if (!BNXT_CHIP_P4_PLUS(bp) && 1602 TPA_START_HASH_TYPE(tpa_start) == 3) 1603 tpa_info->gso_type = SKB_GSO_TCPV6; 1604 tpa_info->rss_hash = 1605 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash); 1606 } else { 1607 tpa_info->hash_type = PKT_HASH_TYPE_NONE; 1608 tpa_info->gso_type = 0; 1609 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n"); 1610 } 1611 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2); 1612 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info); 1613 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) 1614 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1); 1615 else 1616 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1); 1617 tpa_info->agg_count = 0; 1618 1619 rxr->rx_prod = NEXT_RX(prod); 1620 cons = RING_RX(bp, NEXT_RX(cons)); 1621 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); 1622 cons_rx_buf = &rxr->rx_buf_ring[cons]; 1623 1624 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data); 1625 rxr->rx_prod = NEXT_RX(rxr->rx_prod); 1626 cons_rx_buf->data = NULL; 1627} 1628 1629static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs) 1630{ 1631 if (agg_bufs) 1632 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true); 1633} 1634 1635#ifdef CONFIG_INET 1636static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1637{ 1638 struct udphdr *uh = NULL; 1639 1640 if (ip_proto == htons(ETH_P_IP)) { 1641 struct iphdr *iph = (struct iphdr *)skb->data; 1642 1643 if (iph->protocol == IPPROTO_UDP) 1644 uh = (struct udphdr *)(iph + 1); 1645 } else { 1646 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1647 1648 if (iph->nexthdr == IPPROTO_UDP) 1649 uh = (struct udphdr *)(iph + 1); 1650 } 1651 if (uh) { 1652 if (uh->check) 1653 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1654 else 1655 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1656 } 1657} 1658#endif 1659 1660static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info, 1661 int payload_off, int tcp_ts, 1662 struct sk_buff *skb) 1663{ 1664#ifdef CONFIG_INET 1665 struct tcphdr *th; 1666 int len, nw_off; 1667 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1668 u32 hdr_info = tpa_info->hdr_info; 1669 bool loopback = false; 1670 1671 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1672 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1673 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1674 1675 /* If the packet is an internal loopback packet, the offsets will 1676 * have an extra 4 bytes. 1677 */ 1678 if (inner_mac_off == 4) { 1679 loopback = true; 1680 } else if (inner_mac_off > 4) { 1681 __be16 proto = *((__be16 *)(skb->data + inner_ip_off - 1682 ETH_HLEN - 2)); 1683 1684 /* We only support inner iPv4/ipv6. If we don't see the 1685 * correct protocol ID, it must be a loopback packet where 1686 * the offsets are off by 4. 1687 */ 1688 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6)) 1689 loopback = true; 1690 } 1691 if (loopback) { 1692 /* internal loopback packet, subtract all offsets by 4 */ 1693 inner_ip_off -= 4; 1694 inner_mac_off -= 4; 1695 outer_ip_off -= 4; 1696 } 1697 1698 nw_off = inner_ip_off - ETH_HLEN; 1699 skb_set_network_header(skb, nw_off); 1700 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) { 1701 struct ipv6hdr *iph = ipv6_hdr(skb); 1702 1703 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1704 len = skb->len - skb_transport_offset(skb); 1705 th = tcp_hdr(skb); 1706 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1707 } else { 1708 struct iphdr *iph = ip_hdr(skb); 1709 1710 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1711 len = skb->len - skb_transport_offset(skb); 1712 th = tcp_hdr(skb); 1713 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1714 } 1715 1716 if (inner_mac_off) { /* tunnel */ 1717 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1718 ETH_HLEN - 2)); 1719 1720 bnxt_gro_tunnel(skb, proto); 1721 } 1722#endif 1723 return skb; 1724} 1725 1726static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info, 1727 int payload_off, int tcp_ts, 1728 struct sk_buff *skb) 1729{ 1730#ifdef CONFIG_INET 1731 u16 outer_ip_off, inner_ip_off, inner_mac_off; 1732 u32 hdr_info = tpa_info->hdr_info; 1733 int iphdr_len, nw_off; 1734 1735 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info); 1736 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info); 1737 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info); 1738 1739 nw_off = inner_ip_off - ETH_HLEN; 1740 skb_set_network_header(skb, nw_off); 1741 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ? 1742 sizeof(struct ipv6hdr) : sizeof(struct iphdr); 1743 skb_set_transport_header(skb, nw_off + iphdr_len); 1744 1745 if (inner_mac_off) { /* tunnel */ 1746 __be16 proto = *((__be16 *)(skb->data + outer_ip_off - 1747 ETH_HLEN - 2)); 1748 1749 bnxt_gro_tunnel(skb, proto); 1750 } 1751#endif 1752 return skb; 1753} 1754 1755#define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr)) 1756#define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr)) 1757 1758static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info, 1759 int payload_off, int tcp_ts, 1760 struct sk_buff *skb) 1761{ 1762#ifdef CONFIG_INET 1763 struct tcphdr *th; 1764 int len, nw_off, tcp_opt_len = 0; 1765 1766 if (tcp_ts) 1767 tcp_opt_len = 12; 1768 1769 if (tpa_info->gso_type == SKB_GSO_TCPV4) { 1770 struct iphdr *iph; 1771 1772 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len - 1773 ETH_HLEN; 1774 skb_set_network_header(skb, nw_off); 1775 iph = ip_hdr(skb); 1776 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr)); 1777 len = skb->len - skb_transport_offset(skb); 1778 th = tcp_hdr(skb); 1779 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0); 1780 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) { 1781 struct ipv6hdr *iph; 1782 1783 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len - 1784 ETH_HLEN; 1785 skb_set_network_header(skb, nw_off); 1786 iph = ipv6_hdr(skb); 1787 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr)); 1788 len = skb->len - skb_transport_offset(skb); 1789 th = tcp_hdr(skb); 1790 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0); 1791 } else { 1792 dev_kfree_skb_any(skb); 1793 return NULL; 1794 } 1795 1796 if (nw_off) /* tunnel */ 1797 bnxt_gro_tunnel(skb, skb->protocol); 1798#endif 1799 return skb; 1800} 1801 1802static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp, 1803 struct bnxt_tpa_info *tpa_info, 1804 struct rx_tpa_end_cmp *tpa_end, 1805 struct rx_tpa_end_cmp_ext *tpa_end1, 1806 struct sk_buff *skb) 1807{ 1808#ifdef CONFIG_INET 1809 int payload_off; 1810 u16 segs; 1811 1812 segs = TPA_END_TPA_SEGS(tpa_end); 1813 if (segs == 1) 1814 return skb; 1815 1816 NAPI_GRO_CB(skb)->count = segs; 1817 skb_shinfo(skb)->gso_size = 1818 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len); 1819 skb_shinfo(skb)->gso_type = tpa_info->gso_type; 1820 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 1821 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1); 1822 else 1823 payload_off = TPA_END_PAYLOAD_OFF(tpa_end); 1824 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb); 1825 if (likely(skb)) 1826 tcp_gro_complete(skb); 1827#endif 1828 return skb; 1829} 1830 1831/* Given the cfa_code of a received packet determine which 1832 * netdev (vf-rep or PF) the packet is destined to. 1833 */ 1834static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code) 1835{ 1836 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code); 1837 1838 /* if vf-rep dev is NULL, it must belong to the PF */ 1839 return dev ? dev : bp->dev; 1840} 1841 1842static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp, 1843 struct bnxt_cp_ring_info *cpr, 1844 u32 *raw_cons, 1845 struct rx_tpa_end_cmp *tpa_end, 1846 struct rx_tpa_end_cmp_ext *tpa_end1, 1847 u8 *event) 1848{ 1849 struct bnxt_napi *bnapi = cpr->bnapi; 1850 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 1851 struct net_device *dev = bp->dev; 1852 u8 *data_ptr, agg_bufs; 1853 unsigned int len; 1854 struct bnxt_tpa_info *tpa_info; 1855 dma_addr_t mapping; 1856 struct sk_buff *skb; 1857 u16 idx = 0, agg_id; 1858 void *data; 1859 bool gro; 1860 1861 if (unlikely(bnapi->in_reset)) { 1862 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end); 1863 1864 if (rc < 0) 1865 return ERR_PTR(-EBUSY); 1866 return NULL; 1867 } 1868 1869 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 1870 agg_id = TPA_END_AGG_ID_P5(tpa_end); 1871 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 1872 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1); 1873 tpa_info = &rxr->rx_tpa[agg_id]; 1874 if (unlikely(agg_bufs != tpa_info->agg_count)) { 1875 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n", 1876 agg_bufs, tpa_info->agg_count); 1877 agg_bufs = tpa_info->agg_count; 1878 } 1879 tpa_info->agg_count = 0; 1880 *event |= BNXT_AGG_EVENT; 1881 bnxt_free_agg_idx(rxr, agg_id); 1882 idx = agg_id; 1883 gro = !!(bp->flags & BNXT_FLAG_GRO); 1884 } else { 1885 agg_id = TPA_END_AGG_ID(tpa_end); 1886 agg_bufs = TPA_END_AGG_BUFS(tpa_end); 1887 tpa_info = &rxr->rx_tpa[agg_id]; 1888 idx = RING_CMP(*raw_cons); 1889 if (agg_bufs) { 1890 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons)) 1891 return ERR_PTR(-EBUSY); 1892 1893 *event |= BNXT_AGG_EVENT; 1894 idx = NEXT_CMP(idx); 1895 } 1896 gro = !!TPA_END_GRO(tpa_end); 1897 } 1898 data = tpa_info->data; 1899 data_ptr = tpa_info->data_ptr; 1900 prefetch(data_ptr); 1901 len = tpa_info->len; 1902 mapping = tpa_info->mapping; 1903 1904 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) { 1905 bnxt_abort_tpa(cpr, idx, agg_bufs); 1906 if (agg_bufs > MAX_SKB_FRAGS) 1907 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n", 1908 agg_bufs, (int)MAX_SKB_FRAGS); 1909 return NULL; 1910 } 1911 1912 if (len <= bp->rx_copybreak) { 1913 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping); 1914 if (!skb) { 1915 bnxt_abort_tpa(cpr, idx, agg_bufs); 1916 cpr->sw_stats->rx.rx_oom_discards += 1; 1917 return NULL; 1918 } 1919 } else { 1920 u8 *new_data; 1921 dma_addr_t new_mapping; 1922 1923 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, rxr, 1924 GFP_ATOMIC); 1925 if (!new_data) { 1926 bnxt_abort_tpa(cpr, idx, agg_bufs); 1927 cpr->sw_stats->rx.rx_oom_discards += 1; 1928 return NULL; 1929 } 1930 1931 tpa_info->data = new_data; 1932 tpa_info->data_ptr = new_data + bp->rx_offset; 1933 tpa_info->mapping = new_mapping; 1934 1935 skb = napi_build_skb(data, bp->rx_buf_size); 1936 dma_sync_single_for_cpu(&bp->pdev->dev, mapping, 1937 bp->rx_buf_use_size, bp->rx_dir); 1938 1939 if (!skb) { 1940 page_pool_free_va(rxr->head_pool, data, true); 1941 bnxt_abort_tpa(cpr, idx, agg_bufs); 1942 cpr->sw_stats->rx.rx_oom_discards += 1; 1943 return NULL; 1944 } 1945 skb_mark_for_recycle(skb); 1946 skb_reserve(skb, bp->rx_offset); 1947 skb_put(skb, len); 1948 } 1949 1950 if (agg_bufs) { 1951 skb = bnxt_rx_agg_netmems_skb(bp, cpr, skb, idx, agg_bufs, 1952 true); 1953 if (!skb) { 1954 /* Page reuse already handled by bnxt_rx_pages(). */ 1955 cpr->sw_stats->rx.rx_oom_discards += 1; 1956 return NULL; 1957 } 1958 } 1959 1960 if (tpa_info->cfa_code_valid) 1961 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code); 1962 skb->protocol = eth_type_trans(skb, dev); 1963 1964 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE) 1965 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type); 1966 1967 if (tpa_info->vlan_valid && 1968 (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) { 1969 __be16 vlan_proto = htons(tpa_info->metadata >> 1970 RX_CMP_FLAGS2_METADATA_TPID_SFT); 1971 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK; 1972 1973 if (eth_type_vlan(vlan_proto)) { 1974 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 1975 } else { 1976 dev_kfree_skb(skb); 1977 return NULL; 1978 } 1979 } 1980 1981 skb_checksum_none_assert(skb); 1982 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) { 1983 skb->ip_summed = CHECKSUM_UNNECESSARY; 1984 skb->csum_level = 1985 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3; 1986 } 1987 1988 if (gro) 1989 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb); 1990 1991 return skb; 1992} 1993 1994static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 1995 struct rx_agg_cmp *rx_agg) 1996{ 1997 u16 agg_id = TPA_AGG_AGG_ID(rx_agg); 1998 struct bnxt_tpa_info *tpa_info; 1999 2000 agg_id = bnxt_lookup_agg_idx(rxr, agg_id); 2001 tpa_info = &rxr->rx_tpa[agg_id]; 2002 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS); 2003 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg; 2004} 2005 2006static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi, 2007 struct sk_buff *skb) 2008{ 2009 skb_mark_for_recycle(skb); 2010 2011 if (skb->dev != bp->dev) { 2012 /* this packet belongs to a vf-rep */ 2013 bnxt_vf_rep_rx(bp, skb); 2014 return; 2015 } 2016 skb_record_rx_queue(skb, bnapi->index); 2017 napi_gro_receive(&bnapi->napi, skb); 2018} 2019 2020static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags, 2021 struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts) 2022{ 2023 u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp); 2024 2025 if (BNXT_PTP_RX_TS_VALID(flags)) 2026 goto ts_valid; 2027 if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags)) 2028 return false; 2029 2030ts_valid: 2031 *cmpl_ts = ts; 2032 return true; 2033} 2034 2035static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type, 2036 struct rx_cmp *rxcmp, 2037 struct rx_cmp_ext *rxcmp1) 2038{ 2039 __be16 vlan_proto; 2040 u16 vtag; 2041 2042 if (cmp_type == CMP_TYPE_RX_L2_CMP) { 2043 __le32 flags2 = rxcmp1->rx_cmp_flags2; 2044 u32 meta_data; 2045 2046 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN))) 2047 return skb; 2048 2049 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data); 2050 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK; 2051 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT); 2052 if (eth_type_vlan(vlan_proto)) 2053 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 2054 else 2055 goto vlan_err; 2056 } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 2057 if (RX_CMP_VLAN_VALID(rxcmp)) { 2058 u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp); 2059 2060 if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q) 2061 vlan_proto = htons(ETH_P_8021Q); 2062 else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD) 2063 vlan_proto = htons(ETH_P_8021AD); 2064 else 2065 goto vlan_err; 2066 vtag = RX_CMP_METADATA0_TCI(rxcmp1); 2067 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag); 2068 } 2069 } 2070 return skb; 2071vlan_err: 2072 skb_mark_for_recycle(skb); 2073 dev_kfree_skb(skb); 2074 return NULL; 2075} 2076 2077static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp, 2078 struct rx_cmp *rxcmp) 2079{ 2080 u8 ext_op; 2081 2082 ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp); 2083 switch (ext_op) { 2084 case EXT_OP_INNER_4: 2085 case EXT_OP_OUTER_4: 2086 case EXT_OP_INNFL_3: 2087 case EXT_OP_OUTFL_3: 2088 return PKT_HASH_TYPE_L4; 2089 default: 2090 return PKT_HASH_TYPE_L3; 2091 } 2092} 2093 2094/* returns the following: 2095 * 1 - 1 packet successfully received 2096 * 0 - successful TPA_START, packet not completed yet 2097 * -EBUSY - completion ring does not have all the agg buffers yet 2098 * -ENOMEM - packet aborted due to out of memory 2099 * -EIO - packet aborted due to hw error indicated in BD 2100 */ 2101static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 2102 u32 *raw_cons, u8 *event) 2103{ 2104 struct bnxt_napi *bnapi = cpr->bnapi; 2105 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 2106 struct net_device *dev = bp->dev; 2107 struct rx_cmp *rxcmp; 2108 struct rx_cmp_ext *rxcmp1; 2109 u32 tmp_raw_cons = *raw_cons; 2110 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons); 2111 struct skb_shared_info *sinfo; 2112 struct bnxt_sw_rx_bd *rx_buf; 2113 unsigned int len; 2114 u8 *data_ptr, agg_bufs, cmp_type; 2115 bool xdp_active = false; 2116 dma_addr_t dma_addr; 2117 struct sk_buff *skb; 2118 struct xdp_buff xdp; 2119 u32 flags, misc; 2120 u32 cmpl_ts; 2121 void *data; 2122 int rc = 0; 2123 2124 rxcmp = (struct rx_cmp *) 2125 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2126 2127 cmp_type = RX_CMP_TYPE(rxcmp); 2128 2129 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) { 2130 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp); 2131 goto next_rx_no_prod_no_len; 2132 } 2133 2134 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 2135 cp_cons = RING_CMP(tmp_raw_cons); 2136 rxcmp1 = (struct rx_cmp_ext *) 2137 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2138 2139 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2140 return -EBUSY; 2141 2142 /* The valid test of the entry must be done first before 2143 * reading any further. 2144 */ 2145 dma_rmb(); 2146 prod = rxr->rx_prod; 2147 2148 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP || 2149 cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) { 2150 bnxt_tpa_start(bp, rxr, cmp_type, 2151 (struct rx_tpa_start_cmp *)rxcmp, 2152 (struct rx_tpa_start_cmp_ext *)rxcmp1); 2153 2154 *event |= BNXT_RX_EVENT; 2155 goto next_rx_no_prod_no_len; 2156 2157 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2158 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons, 2159 (struct rx_tpa_end_cmp *)rxcmp, 2160 (struct rx_tpa_end_cmp_ext *)rxcmp1, event); 2161 2162 if (IS_ERR(skb)) 2163 return -EBUSY; 2164 2165 rc = -ENOMEM; 2166 if (likely(skb)) { 2167 bnxt_deliver_skb(bp, bnapi, skb); 2168 rc = 1; 2169 } 2170 *event |= BNXT_RX_EVENT; 2171 goto next_rx_no_prod_no_len; 2172 } 2173 2174 cons = rxcmp->rx_cmp_opaque; 2175 if (unlikely(cons != rxr->rx_next_cons)) { 2176 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp); 2177 2178 /* 0xffff is forced error, don't print it */ 2179 if (rxr->rx_next_cons != 0xffff) 2180 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n", 2181 cons, rxr->rx_next_cons); 2182 bnxt_sched_reset_rxr(bp, rxr); 2183 if (rc1) 2184 return rc1; 2185 goto next_rx_no_prod_no_len; 2186 } 2187 rx_buf = &rxr->rx_buf_ring[cons]; 2188 data = rx_buf->data; 2189 data_ptr = rx_buf->data_ptr; 2190 prefetch(data_ptr); 2191 2192 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1); 2193 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT; 2194 2195 if (agg_bufs) { 2196 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons)) 2197 return -EBUSY; 2198 2199 cp_cons = NEXT_CMP(cp_cons); 2200 *event |= BNXT_AGG_EVENT; 2201 } 2202 *event |= BNXT_RX_EVENT; 2203 2204 rx_buf->data = NULL; 2205 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) { 2206 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2); 2207 2208 bnxt_reuse_rx_data(rxr, cons, data); 2209 if (agg_bufs) 2210 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, 2211 false); 2212 2213 rc = -EIO; 2214 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) { 2215 bnapi->cp_ring.sw_stats->rx.rx_buf_errors++; 2216 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 2217 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) { 2218 netdev_warn_once(bp->dev, "RX buffer error %x\n", 2219 rx_err); 2220 bnxt_sched_reset_rxr(bp, rxr); 2221 } 2222 } 2223 goto next_rx_no_len; 2224 } 2225 2226 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type); 2227 len = flags >> RX_CMP_LEN_SHIFT; 2228 dma_addr = rx_buf->mapping; 2229 2230 if (bnxt_xdp_attached(bp, rxr)) { 2231 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); 2232 if (agg_bufs) { 2233 u32 frag_len = bnxt_rx_agg_netmems_xdp(bp, cpr, &xdp, 2234 cp_cons, 2235 agg_bufs, 2236 false); 2237 if (!frag_len) 2238 goto oom_next_rx; 2239 2240 } 2241 xdp_active = true; 2242 } 2243 2244 if (xdp_active) { 2245 if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) { 2246 rc = 1; 2247 goto next_rx; 2248 } 2249 if (xdp_buff_has_frags(&xdp)) { 2250 sinfo = xdp_get_shared_info_from_buff(&xdp); 2251 agg_bufs = sinfo->nr_frags; 2252 } else { 2253 agg_bufs = 0; 2254 } 2255 } 2256 2257 if (len <= bp->rx_copybreak) { 2258 if (!xdp_active) 2259 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); 2260 else 2261 skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr); 2262 bnxt_reuse_rx_data(rxr, cons, data); 2263 if (!skb) { 2264 if (agg_bufs) { 2265 if (!xdp_active) 2266 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, 2267 agg_bufs, false); 2268 else 2269 bnxt_xdp_buff_frags_free(rxr, &xdp); 2270 } 2271 goto oom_next_rx; 2272 } 2273 } else { 2274 u32 payload; 2275 2276 if (rx_buf->data_ptr == data_ptr) 2277 payload = misc & RX_CMP_PAYLOAD_OFFSET; 2278 else 2279 payload = 0; 2280 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr, 2281 payload | len); 2282 if (!skb) 2283 goto oom_next_rx; 2284 } 2285 2286 if (agg_bufs) { 2287 if (!xdp_active) { 2288 skb = bnxt_rx_agg_netmems_skb(bp, cpr, skb, cp_cons, 2289 agg_bufs, false); 2290 if (!skb) 2291 goto oom_next_rx; 2292 } else { 2293 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, 2294 rxr->page_pool, &xdp); 2295 if (!skb) { 2296 /* we should be able to free the old skb here */ 2297 bnxt_xdp_buff_frags_free(rxr, &xdp); 2298 goto oom_next_rx; 2299 } 2300 } 2301 } 2302 2303 if (RX_CMP_HASH_VALID(rxcmp)) { 2304 enum pkt_hash_types type; 2305 2306 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 2307 type = bnxt_rss_ext_op(bp, rxcmp); 2308 } else { 2309 u32 itypes = RX_CMP_ITYPES(rxcmp); 2310 2311 if (itypes == RX_CMP_FLAGS_ITYPE_TCP || 2312 itypes == RX_CMP_FLAGS_ITYPE_UDP) 2313 type = PKT_HASH_TYPE_L4; 2314 else 2315 type = PKT_HASH_TYPE_L3; 2316 } 2317 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type); 2318 } 2319 2320 if (cmp_type == CMP_TYPE_RX_L2_CMP) 2321 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1)); 2322 skb->protocol = eth_type_trans(skb, dev); 2323 2324 if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) { 2325 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1); 2326 if (!skb) 2327 goto next_rx; 2328 } 2329 2330 skb_checksum_none_assert(skb); 2331 if (RX_CMP_L4_CS_OK(rxcmp1)) { 2332 if (dev->features & NETIF_F_RXCSUM) { 2333 skb->ip_summed = CHECKSUM_UNNECESSARY; 2334 skb->csum_level = RX_CMP_ENCAP(rxcmp1); 2335 } 2336 } else { 2337 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) { 2338 if (dev->features & NETIF_F_RXCSUM) 2339 bnapi->cp_ring.sw_stats->rx.rx_l4_csum_errors++; 2340 } 2341 } 2342 2343 if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) { 2344 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 2345 u64 ns, ts; 2346 2347 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) { 2348 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2349 2350 ns = bnxt_timecounter_cyc2time(ptp, ts); 2351 memset(skb_hwtstamps(skb), 0, 2352 sizeof(*skb_hwtstamps(skb))); 2353 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); 2354 } 2355 } 2356 } 2357 bnxt_deliver_skb(bp, bnapi, skb); 2358 rc = 1; 2359 2360next_rx: 2361 cpr->rx_packets += 1; 2362 cpr->rx_bytes += len; 2363 2364next_rx_no_len: 2365 rxr->rx_prod = NEXT_RX(prod); 2366 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons)); 2367 2368next_rx_no_prod_no_len: 2369 *raw_cons = tmp_raw_cons; 2370 2371 return rc; 2372 2373oom_next_rx: 2374 cpr->sw_stats->rx.rx_oom_discards += 1; 2375 rc = -ENOMEM; 2376 goto next_rx; 2377} 2378 2379/* In netpoll mode, if we are using a combined completion ring, we need to 2380 * discard the rx packets and recycle the buffers. 2381 */ 2382static int bnxt_force_rx_discard(struct bnxt *bp, 2383 struct bnxt_cp_ring_info *cpr, 2384 u32 *raw_cons, u8 *event) 2385{ 2386 u32 tmp_raw_cons = *raw_cons; 2387 struct rx_cmp_ext *rxcmp1; 2388 struct rx_cmp *rxcmp; 2389 u16 cp_cons; 2390 u8 cmp_type; 2391 int rc; 2392 2393 cp_cons = RING_CMP(tmp_raw_cons); 2394 rxcmp = (struct rx_cmp *) 2395 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2396 2397 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons); 2398 cp_cons = RING_CMP(tmp_raw_cons); 2399 rxcmp1 = (struct rx_cmp_ext *) 2400 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 2401 2402 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 2403 return -EBUSY; 2404 2405 /* The valid test of the entry must be done first before 2406 * reading any further. 2407 */ 2408 dma_rmb(); 2409 cmp_type = RX_CMP_TYPE(rxcmp); 2410 if (cmp_type == CMP_TYPE_RX_L2_CMP || 2411 cmp_type == CMP_TYPE_RX_L2_V3_CMP) { 2412 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 2413 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 2414 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) { 2415 struct rx_tpa_end_cmp_ext *tpa_end1; 2416 2417 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1; 2418 tpa_end1->rx_tpa_end_cmp_errors_v2 |= 2419 cpu_to_le32(RX_TPA_END_CMP_ERRORS); 2420 } 2421 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event); 2422 if (rc && rc != -EBUSY) 2423 cpr->sw_stats->rx.rx_netpoll_discards += 1; 2424 return rc; 2425} 2426 2427u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx) 2428{ 2429 struct bnxt_fw_health *fw_health = bp->fw_health; 2430 u32 reg = fw_health->regs[reg_idx]; 2431 u32 reg_type, reg_off, val = 0; 2432 2433 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 2434 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 2435 switch (reg_type) { 2436 case BNXT_FW_HEALTH_REG_TYPE_CFG: 2437 pci_read_config_dword(bp->pdev, reg_off, &val); 2438 break; 2439 case BNXT_FW_HEALTH_REG_TYPE_GRC: 2440 reg_off = fw_health->mapped_regs[reg_idx]; 2441 fallthrough; 2442 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 2443 val = readl(bp->bar0 + reg_off); 2444 break; 2445 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 2446 val = readl(bp->bar1 + reg_off); 2447 break; 2448 } 2449 if (reg_idx == BNXT_FW_RESET_INPROG_REG) 2450 val &= fw_health->fw_reset_inprog_reg_mask; 2451 return val; 2452} 2453 2454static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id) 2455{ 2456 int i; 2457 2458 for (i = 0; i < bp->rx_nr_rings; i++) { 2459 u16 grp_idx = bp->rx_ring[i].bnapi->index; 2460 struct bnxt_ring_grp_info *grp_info; 2461 2462 grp_info = &bp->grp_info[grp_idx]; 2463 if (grp_info->agg_fw_ring_id == ring_id) 2464 return grp_idx; 2465 } 2466 return INVALID_HW_RING_ID; 2467} 2468 2469static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info) 2470{ 2471 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2472 2473 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) 2474 return link_info->force_link_speed2; 2475 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4) 2476 return link_info->force_pam4_link_speed; 2477 return link_info->force_link_speed; 2478} 2479 2480static void bnxt_set_force_speed(struct bnxt_link_info *link_info) 2481{ 2482 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2483 2484 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2485 link_info->req_link_speed = link_info->force_link_speed2; 2486 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2487 switch (link_info->req_link_speed) { 2488 case BNXT_LINK_SPEED_50GB_PAM4: 2489 case BNXT_LINK_SPEED_100GB_PAM4: 2490 case BNXT_LINK_SPEED_200GB_PAM4: 2491 case BNXT_LINK_SPEED_400GB_PAM4: 2492 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 2493 break; 2494 case BNXT_LINK_SPEED_100GB_PAM4_112: 2495 case BNXT_LINK_SPEED_200GB_PAM4_112: 2496 case BNXT_LINK_SPEED_400GB_PAM4_112: 2497 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112; 2498 break; 2499 default: 2500 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2501 } 2502 return; 2503 } 2504 link_info->req_link_speed = link_info->force_link_speed; 2505 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ; 2506 if (link_info->force_pam4_link_speed) { 2507 link_info->req_link_speed = link_info->force_pam4_link_speed; 2508 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4; 2509 } 2510} 2511 2512static void bnxt_set_auto_speed(struct bnxt_link_info *link_info) 2513{ 2514 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2515 2516 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2517 link_info->advertising = link_info->auto_link_speeds2; 2518 return; 2519 } 2520 link_info->advertising = link_info->auto_link_speeds; 2521 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds; 2522} 2523 2524static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info) 2525{ 2526 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2527 2528 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2529 if (link_info->req_link_speed != link_info->force_link_speed2) 2530 return true; 2531 return false; 2532 } 2533 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ && 2534 link_info->req_link_speed != link_info->force_link_speed) 2535 return true; 2536 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 && 2537 link_info->req_link_speed != link_info->force_pam4_link_speed) 2538 return true; 2539 return false; 2540} 2541 2542static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info) 2543{ 2544 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 2545 2546 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 2547 if (link_info->advertising != link_info->auto_link_speeds2) 2548 return true; 2549 return false; 2550 } 2551 if (link_info->advertising != link_info->auto_link_speeds || 2552 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds) 2553 return true; 2554 return false; 2555} 2556 2557bool bnxt_bs_trace_avail(struct bnxt *bp, u16 type) 2558{ 2559 u32 flags = bp->ctx->ctx_arr[type].flags; 2560 2561 return (flags & BNXT_CTX_MEM_TYPE_VALID) && 2562 ((flags & BNXT_CTX_MEM_FW_TRACE) || 2563 (flags & BNXT_CTX_MEM_FW_BIN_TRACE)); 2564} 2565 2566static void bnxt_bs_trace_init(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm) 2567{ 2568 u32 mem_size, pages, rem_bytes, magic_byte_offset; 2569 u16 trace_type = bnxt_bstore_to_trace[ctxm->type]; 2570 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; 2571 struct bnxt_ring_mem_info *rmem, *rmem_pg_tbl; 2572 struct bnxt_bs_trace_info *bs_trace; 2573 int last_pg; 2574 2575 if (ctxm->instance_bmap && ctxm->instance_bmap > 1) 2576 return; 2577 2578 mem_size = ctxm->max_entries * ctxm->entry_size; 2579 rem_bytes = mem_size % BNXT_PAGE_SIZE; 2580 pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 2581 2582 last_pg = (pages - 1) & (MAX_CTX_PAGES - 1); 2583 magic_byte_offset = (rem_bytes ? rem_bytes : BNXT_PAGE_SIZE) - 1; 2584 2585 rmem = &ctx_pg[0].ring_mem; 2586 bs_trace = &bp->bs_trace[trace_type]; 2587 bs_trace->ctx_type = ctxm->type; 2588 bs_trace->trace_type = trace_type; 2589 if (pages > MAX_CTX_PAGES) { 2590 int last_pg_dir = rmem->nr_pages - 1; 2591 2592 rmem_pg_tbl = &ctx_pg[0].ctx_pg_tbl[last_pg_dir]->ring_mem; 2593 bs_trace->magic_byte = rmem_pg_tbl->pg_arr[last_pg]; 2594 } else { 2595 bs_trace->magic_byte = rmem->pg_arr[last_pg]; 2596 } 2597 bs_trace->magic_byte += magic_byte_offset; 2598 *bs_trace->magic_byte = BNXT_TRACE_BUF_MAGIC_BYTE; 2599} 2600 2601#define BNXT_EVENT_BUF_PRODUCER_TYPE(data1) \ 2602 (((data1) & ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_MASK) >>\ 2603 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SFT) 2604 2605#define BNXT_EVENT_BUF_PRODUCER_OFFSET(data2) \ 2606 (((data2) & \ 2607 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_MASK) >>\ 2608 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_SFT) 2609 2610#define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \ 2611 ((data2) & \ 2612 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK) 2613 2614#define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \ 2615 (((data2) & \ 2616 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\ 2617 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT) 2618 2619#define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \ 2620 ((data1) & \ 2621 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK) 2622 2623#define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \ 2624 (((data1) & \ 2625 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\ 2626 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING) 2627 2628/* Return true if the workqueue has to be scheduled */ 2629static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2) 2630{ 2631 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); 2632 2633 switch (err_type) { 2634 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: 2635 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n", 2636 BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); 2637 break; 2638 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: 2639 netdev_warn(bp->dev, "Pause Storm detected!\n"); 2640 break; 2641 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: 2642 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n"); 2643 break; 2644 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: { 2645 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1); 2646 char *threshold_type; 2647 bool notify = false; 2648 char *dir_str; 2649 2650 switch (type) { 2651 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN: 2652 threshold_type = "warning"; 2653 break; 2654 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL: 2655 threshold_type = "critical"; 2656 break; 2657 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL: 2658 threshold_type = "fatal"; 2659 break; 2660 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN: 2661 threshold_type = "shutdown"; 2662 break; 2663 default: 2664 netdev_err(bp->dev, "Unknown Thermal threshold type event\n"); 2665 return false; 2666 } 2667 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) { 2668 dir_str = "above"; 2669 notify = true; 2670 } else { 2671 dir_str = "below"; 2672 } 2673 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n", 2674 dir_str, threshold_type); 2675 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n", 2676 BNXT_EVENT_THERMAL_CURRENT_TEMP(data2), 2677 BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)); 2678 if (notify) { 2679 bp->thermal_threshold_type = type; 2680 set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event); 2681 return true; 2682 } 2683 return false; 2684 } 2685 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED: 2686 netdev_warn(bp->dev, "Speed change not supported with dual rate transceivers on this board\n"); 2687 break; 2688 default: 2689 netdev_err(bp->dev, "FW reported unknown error type %u\n", 2690 err_type); 2691 break; 2692 } 2693 return false; 2694} 2695 2696#define BNXT_GET_EVENT_PORT(data) \ 2697 ((data) & \ 2698 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK) 2699 2700#define BNXT_EVENT_RING_TYPE(data2) \ 2701 ((data2) & \ 2702 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK) 2703 2704#define BNXT_EVENT_RING_TYPE_RX(data2) \ 2705 (BNXT_EVENT_RING_TYPE(data2) == \ 2706 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX) 2707 2708#define BNXT_EVENT_PHC_EVENT_TYPE(data1) \ 2709 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\ 2710 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT) 2711 2712#define BNXT_EVENT_PHC_RTC_UPDATE(data1) \ 2713 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\ 2714 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT) 2715 2716#define BNXT_PHC_BITS 48 2717 2718static int bnxt_async_event_process(struct bnxt *bp, 2719 struct hwrm_async_event_cmpl *cmpl) 2720{ 2721 u16 event_id = le16_to_cpu(cmpl->event_id); 2722 u32 data1 = le32_to_cpu(cmpl->event_data1); 2723 u32 data2 = le32_to_cpu(cmpl->event_data2); 2724 2725 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n", 2726 event_id, data1, data2); 2727 2728 /* TODO CHIMP_FW: Define event id's for link change, error etc */ 2729 switch (event_id) { 2730 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: { 2731 struct bnxt_link_info *link_info = &bp->link_info; 2732 2733 if (BNXT_VF(bp)) 2734 goto async_event_process_exit; 2735 2736 /* print unsupported speed warning in forced speed mode only */ 2737 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) && 2738 (data1 & 0x20000)) { 2739 u16 fw_speed = bnxt_get_force_speed(link_info); 2740 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed); 2741 2742 if (speed != SPEED_UNKNOWN) 2743 netdev_warn(bp->dev, "Link speed %d no longer supported\n", 2744 speed); 2745 } 2746 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event); 2747 } 2748 fallthrough; 2749 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: 2750 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: 2751 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event); 2752 fallthrough; 2753 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: 2754 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event); 2755 break; 2756 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: 2757 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event); 2758 break; 2759 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: { 2760 u16 port_id = BNXT_GET_EVENT_PORT(data1); 2761 2762 if (BNXT_VF(bp)) 2763 break; 2764 2765 if (bp->pf.port_id != port_id) 2766 break; 2767 2768 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event); 2769 break; 2770 } 2771 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: 2772 if (BNXT_PF(bp)) 2773 goto async_event_process_exit; 2774 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event); 2775 break; 2776 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { 2777 char *type_str = "Solicited"; 2778 2779 if (!bp->fw_health) 2780 goto async_event_process_exit; 2781 2782 bp->fw_reset_timestamp = jiffies; 2783 bp->fw_reset_min_dsecs = cmpl->timestamp_lo; 2784 if (!bp->fw_reset_min_dsecs) 2785 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; 2786 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi); 2787 if (!bp->fw_reset_max_dsecs) 2788 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; 2789 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { 2790 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state); 2791 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { 2792 type_str = "Fatal"; 2793 bp->fw_health->fatalities++; 2794 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 2795 } else if (data2 && BNXT_FW_STATUS_HEALTHY != 2796 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { 2797 type_str = "Non-fatal"; 2798 bp->fw_health->survivals++; 2799 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 2800 } 2801 netif_warn(bp, hw, bp->dev, 2802 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", 2803 type_str, data1, data2, 2804 bp->fw_reset_min_dsecs * 100, 2805 bp->fw_reset_max_dsecs * 100); 2806 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event); 2807 break; 2808 } 2809 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { 2810 struct bnxt_fw_health *fw_health = bp->fw_health; 2811 char *status_desc = "healthy"; 2812 u32 status; 2813 2814 if (!fw_health) 2815 goto async_event_process_exit; 2816 2817 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { 2818 fw_health->enabled = false; 2819 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n"); 2820 break; 2821 } 2822 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); 2823 fw_health->tmr_multiplier = 2824 DIV_ROUND_UP(fw_health->polling_dsecs * HZ, 2825 bp->current_interval * 10); 2826 fw_health->tmr_counter = fw_health->tmr_multiplier; 2827 if (!fw_health->enabled) 2828 fw_health->last_fw_heartbeat = 2829 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 2830 fw_health->last_fw_reset_cnt = 2831 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 2832 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 2833 if (status != BNXT_FW_STATUS_HEALTHY) 2834 status_desc = "unhealthy"; 2835 netif_info(bp, drv, bp->dev, 2836 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", 2837 fw_health->primary ? "primary" : "backup", status, 2838 status_desc, fw_health->last_fw_reset_cnt); 2839 if (!fw_health->enabled) { 2840 /* Make sure tmr_counter is set and visible to 2841 * bnxt_health_check() before setting enabled to true. 2842 */ 2843 smp_wmb(); 2844 fw_health->enabled = true; 2845 } 2846 goto async_event_process_exit; 2847 } 2848 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION: 2849 netif_notice(bp, hw, bp->dev, 2850 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n", 2851 data1, data2); 2852 goto async_event_process_exit; 2853 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: { 2854 struct bnxt_rx_ring_info *rxr; 2855 u16 grp_idx; 2856 2857 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 2858 goto async_event_process_exit; 2859 2860 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n", 2861 BNXT_EVENT_RING_TYPE(data2), data1); 2862 if (!BNXT_EVENT_RING_TYPE_RX(data2)) 2863 goto async_event_process_exit; 2864 2865 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1); 2866 if (grp_idx == INVALID_HW_RING_ID) { 2867 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n", 2868 data1); 2869 goto async_event_process_exit; 2870 } 2871 rxr = bp->bnapi[grp_idx]->rx_ring; 2872 bnxt_sched_reset_rxr(bp, rxr); 2873 goto async_event_process_exit; 2874 } 2875 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: { 2876 struct bnxt_fw_health *fw_health = bp->fw_health; 2877 2878 netif_notice(bp, hw, bp->dev, 2879 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n", 2880 data1, data2); 2881 if (fw_health) { 2882 fw_health->echo_req_data1 = data1; 2883 fw_health->echo_req_data2 = data2; 2884 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event); 2885 break; 2886 } 2887 goto async_event_process_exit; 2888 } 2889 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: { 2890 bnxt_ptp_pps_event(bp, data1, data2); 2891 goto async_event_process_exit; 2892 } 2893 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { 2894 if (bnxt_event_error_report(bp, data1, data2)) 2895 break; 2896 goto async_event_process_exit; 2897 } 2898 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: { 2899 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) { 2900 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE: 2901 if (BNXT_PTP_USE_RTC(bp)) { 2902 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 2903 unsigned long flags; 2904 u64 ns; 2905 2906 if (!ptp) 2907 goto async_event_process_exit; 2908 2909 bnxt_ptp_update_current_time(bp); 2910 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) << 2911 BNXT_PHC_BITS) | ptp->current_time); 2912 write_seqlock_irqsave(&ptp->ptp_lock, flags); 2913 bnxt_ptp_rtc_timecounter_init(ptp, ns); 2914 write_sequnlock_irqrestore(&ptp->ptp_lock, flags); 2915 } 2916 break; 2917 } 2918 goto async_event_process_exit; 2919 } 2920 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: { 2921 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff; 2922 2923 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED); 2924 goto async_event_process_exit; 2925 } 2926 case ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER: { 2927 u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1); 2928 u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2); 2929 2930 if (type >= ARRAY_SIZE(bp->bs_trace)) 2931 goto async_event_process_exit; 2932 bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset); 2933 goto async_event_process_exit; 2934 } 2935 default: 2936 goto async_event_process_exit; 2937 } 2938 __bnxt_queue_sp_work(bp); 2939async_event_process_exit: 2940 bnxt_ulp_async_events(bp, cmpl); 2941 return 0; 2942} 2943 2944static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) 2945{ 2946 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id; 2947 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; 2948 struct hwrm_fwd_req_cmpl *fwd_req_cmpl = 2949 (struct hwrm_fwd_req_cmpl *)txcmp; 2950 2951 switch (cmpl_type) { 2952 case CMPL_BASE_TYPE_HWRM_DONE: 2953 seq_id = le16_to_cpu(h_cmpl->sequence_id); 2954 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE); 2955 break; 2956 2957 case CMPL_BASE_TYPE_HWRM_FWD_REQ: 2958 vf_id = le16_to_cpu(fwd_req_cmpl->source_id); 2959 2960 if ((vf_id < bp->pf.first_vf_id) || 2961 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) { 2962 netdev_err(bp->dev, "Msg contains invalid VF id %x\n", 2963 vf_id); 2964 return -EINVAL; 2965 } 2966 2967 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap); 2968 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT); 2969 break; 2970 2971 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: 2972 bnxt_async_event_process(bp, 2973 (struct hwrm_async_event_cmpl *)txcmp); 2974 break; 2975 2976 default: 2977 break; 2978 } 2979 2980 return 0; 2981} 2982 2983static bool bnxt_vnic_is_active(struct bnxt *bp) 2984{ 2985 struct bnxt_vnic_info *vnic = &bp->vnic_info[0]; 2986 2987 return vnic->fw_vnic_id != INVALID_HW_RING_ID && vnic->mru > 0; 2988} 2989 2990static irqreturn_t bnxt_msix(int irq, void *dev_instance) 2991{ 2992 struct bnxt_napi *bnapi = dev_instance; 2993 struct bnxt *bp = bnapi->bp; 2994 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 2995 u32 cons = RING_CMP(cpr->cp_raw_cons); 2996 2997 cpr->event_ctr++; 2998 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); 2999 napi_schedule(&bnapi->napi); 3000 return IRQ_HANDLED; 3001} 3002 3003static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 3004{ 3005 u32 raw_cons = cpr->cp_raw_cons; 3006 u16 cons = RING_CMP(raw_cons); 3007 struct tx_cmp *txcmp; 3008 3009 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 3010 3011 return TX_CMP_VALID(txcmp, raw_cons); 3012} 3013 3014static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 3015 int budget) 3016{ 3017 struct bnxt_napi *bnapi = cpr->bnapi; 3018 u32 raw_cons = cpr->cp_raw_cons; 3019 bool flush_xdp = false; 3020 u32 cons; 3021 int rx_pkts = 0; 3022 u8 event = 0; 3023 struct tx_cmp *txcmp; 3024 3025 cpr->has_more_work = 0; 3026 cpr->had_work_done = 1; 3027 while (1) { 3028 u8 cmp_type; 3029 int rc; 3030 3031 cons = RING_CMP(raw_cons); 3032 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 3033 3034 if (!TX_CMP_VALID(txcmp, raw_cons)) 3035 break; 3036 3037 /* The valid test of the entry must be done first before 3038 * reading any further. 3039 */ 3040 dma_rmb(); 3041 cmp_type = TX_CMP_TYPE(txcmp); 3042 if (cmp_type == CMP_TYPE_TX_L2_CMP || 3043 cmp_type == CMP_TYPE_TX_L2_COAL_CMP) { 3044 u32 opaque = txcmp->tx_cmp_opaque; 3045 struct bnxt_tx_ring_info *txr; 3046 u16 tx_freed; 3047 3048 txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)]; 3049 event |= BNXT_TX_CMP_EVENT; 3050 if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP) 3051 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp); 3052 else 3053 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque); 3054 tx_freed = (txr->tx_hw_cons - txr->tx_cons) & 3055 bp->tx_ring_mask; 3056 /* return full budget so NAPI will complete. */ 3057 if (unlikely(tx_freed >= bp->tx_wake_thresh)) { 3058 rx_pkts = budget; 3059 raw_cons = NEXT_RAW_CMP(raw_cons); 3060 if (budget) 3061 cpr->has_more_work = 1; 3062 break; 3063 } 3064 } else if (cmp_type == CMP_TYPE_TX_L2_PKT_TS_CMP) { 3065 bnxt_tx_ts_cmp(bp, bnapi, (struct tx_ts_cmp *)txcmp); 3066 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP && 3067 cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) { 3068 if (likely(budget)) 3069 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 3070 else 3071 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons, 3072 &event); 3073 if (event & BNXT_REDIRECT_EVENT) 3074 flush_xdp = true; 3075 if (likely(rc >= 0)) 3076 rx_pkts += rc; 3077 /* Increment rx_pkts when rc is -ENOMEM to count towards 3078 * the NAPI budget. Otherwise, we may potentially loop 3079 * here forever if we consistently cannot allocate 3080 * buffers. 3081 */ 3082 else if (rc == -ENOMEM && budget) 3083 rx_pkts++; 3084 else if (rc == -EBUSY) /* partial completion */ 3085 break; 3086 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE || 3087 cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ || 3088 cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) { 3089 bnxt_hwrm_handler(bp, txcmp); 3090 } 3091 raw_cons = NEXT_RAW_CMP(raw_cons); 3092 3093 if (rx_pkts && rx_pkts == budget) { 3094 cpr->has_more_work = 1; 3095 break; 3096 } 3097 } 3098 3099 if (flush_xdp) { 3100 xdp_do_flush(); 3101 event &= ~BNXT_REDIRECT_EVENT; 3102 } 3103 3104 if (event & BNXT_TX_EVENT) { 3105 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0]; 3106 u16 prod = txr->tx_prod; 3107 3108 /* Sync BD data before updating doorbell */ 3109 wmb(); 3110 3111 bnxt_db_write_relaxed(bp, &txr->tx_db, prod); 3112 event &= ~BNXT_TX_EVENT; 3113 } 3114 3115 cpr->cp_raw_cons = raw_cons; 3116 bnapi->events |= event; 3117 return rx_pkts; 3118} 3119 3120static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi, 3121 int budget) 3122{ 3123 if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault) 3124 bnapi->tx_int(bp, bnapi, budget); 3125 3126 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) { 3127 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 3128 3129 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 3130 bnapi->events &= ~BNXT_RX_EVENT; 3131 } 3132 if (bnapi->events & BNXT_AGG_EVENT) { 3133 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 3134 3135 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 3136 bnapi->events &= ~BNXT_AGG_EVENT; 3137 } 3138} 3139 3140static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, 3141 int budget) 3142{ 3143 struct bnxt_napi *bnapi = cpr->bnapi; 3144 int rx_pkts; 3145 3146 rx_pkts = __bnxt_poll_work(bp, cpr, budget); 3147 3148 /* ACK completion ring before freeing tx ring and producing new 3149 * buffers in rx/agg rings to prevent overflowing the completion 3150 * ring. 3151 */ 3152 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 3153 3154 __bnxt_poll_work_done(bp, bnapi, budget); 3155 return rx_pkts; 3156} 3157 3158static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget) 3159{ 3160 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 3161 struct bnxt *bp = bnapi->bp; 3162 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3163 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 3164 struct tx_cmp *txcmp; 3165 struct rx_cmp_ext *rxcmp1; 3166 u32 cp_cons, tmp_raw_cons; 3167 u32 raw_cons = cpr->cp_raw_cons; 3168 bool flush_xdp = false; 3169 u32 rx_pkts = 0; 3170 u8 event = 0; 3171 3172 while (1) { 3173 int rc; 3174 3175 cp_cons = RING_CMP(raw_cons); 3176 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 3177 3178 if (!TX_CMP_VALID(txcmp, raw_cons)) 3179 break; 3180 3181 /* The valid test of the entry must be done first before 3182 * reading any further. 3183 */ 3184 dma_rmb(); 3185 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) { 3186 tmp_raw_cons = NEXT_RAW_CMP(raw_cons); 3187 cp_cons = RING_CMP(tmp_raw_cons); 3188 rxcmp1 = (struct rx_cmp_ext *) 3189 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)]; 3190 3191 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons)) 3192 break; 3193 3194 /* force an error to recycle the buffer */ 3195 rxcmp1->rx_cmp_cfa_code_errors_v2 |= 3196 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR); 3197 3198 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event); 3199 if (likely(rc == -EIO) && budget) 3200 rx_pkts++; 3201 else if (rc == -EBUSY) /* partial completion */ 3202 break; 3203 if (event & BNXT_REDIRECT_EVENT) 3204 flush_xdp = true; 3205 } else if (unlikely(TX_CMP_TYPE(txcmp) == 3206 CMPL_BASE_TYPE_HWRM_DONE)) { 3207 bnxt_hwrm_handler(bp, txcmp); 3208 } else { 3209 netdev_err(bp->dev, 3210 "Invalid completion received on special ring\n"); 3211 } 3212 raw_cons = NEXT_RAW_CMP(raw_cons); 3213 3214 if (rx_pkts == budget) 3215 break; 3216 } 3217 3218 cpr->cp_raw_cons = raw_cons; 3219 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons); 3220 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 3221 3222 if (event & BNXT_AGG_EVENT) 3223 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 3224 if (flush_xdp) 3225 xdp_do_flush(); 3226 3227 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) { 3228 napi_complete_done(napi, rx_pkts); 3229 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3230 } 3231 return rx_pkts; 3232} 3233 3234static int bnxt_poll(struct napi_struct *napi, int budget) 3235{ 3236 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 3237 struct bnxt *bp = bnapi->bp; 3238 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3239 int work_done = 0; 3240 3241 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 3242 napi_complete(napi); 3243 return 0; 3244 } 3245 while (1) { 3246 work_done += bnxt_poll_work(bp, cpr, budget - work_done); 3247 3248 if (work_done >= budget) { 3249 if (!budget) 3250 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3251 break; 3252 } 3253 3254 if (!bnxt_has_work(bp, cpr)) { 3255 if (napi_complete_done(napi, work_done)) 3256 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons); 3257 break; 3258 } 3259 } 3260 if ((bp->flags & BNXT_FLAG_DIM) && bnxt_vnic_is_active(bp)) { 3261 struct dim_sample dim_sample = {}; 3262 3263 dim_update_sample(cpr->event_ctr, 3264 cpr->rx_packets, 3265 cpr->rx_bytes, 3266 &dim_sample); 3267 net_dim(&cpr->dim, &dim_sample); 3268 } 3269 return work_done; 3270} 3271 3272static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 3273{ 3274 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3275 int i, work_done = 0; 3276 3277 for (i = 0; i < cpr->cp_ring_count; i++) { 3278 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i]; 3279 3280 if (cpr2->had_nqe_notify) { 3281 work_done += __bnxt_poll_work(bp, cpr2, 3282 budget - work_done); 3283 cpr->has_more_work |= cpr2->has_more_work; 3284 } 3285 } 3286 return work_done; 3287} 3288 3289static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi, 3290 u64 dbr_type, int budget) 3291{ 3292 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3293 int i; 3294 3295 for (i = 0; i < cpr->cp_ring_count; i++) { 3296 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i]; 3297 struct bnxt_db_info *db; 3298 3299 if (cpr2->had_work_done) { 3300 u32 tgl = 0; 3301 3302 if (dbr_type == DBR_TYPE_CQ_ARMALL) { 3303 cpr2->had_nqe_notify = 0; 3304 tgl = cpr2->toggle; 3305 } 3306 db = &cpr2->cp_db; 3307 bnxt_writeq(bp, 3308 db->db_key64 | dbr_type | DB_TOGGLE(tgl) | 3309 DB_RING_IDX(db, cpr2->cp_raw_cons), 3310 db->doorbell); 3311 cpr2->had_work_done = 0; 3312 } 3313 } 3314 __bnxt_poll_work_done(bp, bnapi, budget); 3315} 3316 3317static int bnxt_poll_p5(struct napi_struct *napi, int budget) 3318{ 3319 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi); 3320 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 3321 struct bnxt_cp_ring_info *cpr_rx; 3322 u32 raw_cons = cpr->cp_raw_cons; 3323 struct bnxt *bp = bnapi->bp; 3324 struct nqe_cn *nqcmp; 3325 int work_done = 0; 3326 u32 cons; 3327 3328 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) { 3329 napi_complete(napi); 3330 return 0; 3331 } 3332 if (cpr->has_more_work) { 3333 cpr->has_more_work = 0; 3334 work_done = __bnxt_poll_cqs(bp, bnapi, budget); 3335 } 3336 while (1) { 3337 u16 type; 3338 3339 cons = RING_CMP(raw_cons); 3340 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)]; 3341 3342 if (!NQ_CMP_VALID(nqcmp, raw_cons)) { 3343 if (cpr->has_more_work) 3344 break; 3345 3346 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, 3347 budget); 3348 cpr->cp_raw_cons = raw_cons; 3349 if (napi_complete_done(napi, work_done)) 3350 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, 3351 cpr->cp_raw_cons); 3352 goto poll_done; 3353 } 3354 3355 /* The valid test of the entry must be done first before 3356 * reading any further. 3357 */ 3358 dma_rmb(); 3359 3360 type = le16_to_cpu(nqcmp->type); 3361 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) { 3362 u32 idx = le32_to_cpu(nqcmp->cq_handle_low); 3363 u32 cq_type = BNXT_NQ_HDL_TYPE(idx); 3364 struct bnxt_cp_ring_info *cpr2; 3365 3366 /* No more budget for RX work */ 3367 if (budget && work_done >= budget && 3368 cq_type == BNXT_NQ_HDL_TYPE_RX) 3369 break; 3370 3371 idx = BNXT_NQ_HDL_IDX(idx); 3372 cpr2 = &cpr->cp_ring_arr[idx]; 3373 cpr2->had_nqe_notify = 1; 3374 cpr2->toggle = NQE_CN_TOGGLE(type); 3375 work_done += __bnxt_poll_work(bp, cpr2, 3376 budget - work_done); 3377 cpr->has_more_work |= cpr2->has_more_work; 3378 } else { 3379 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp); 3380 } 3381 raw_cons = NEXT_RAW_CMP(raw_cons); 3382 } 3383 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget); 3384 if (raw_cons != cpr->cp_raw_cons) { 3385 cpr->cp_raw_cons = raw_cons; 3386 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons); 3387 } 3388poll_done: 3389 cpr_rx = &cpr->cp_ring_arr[0]; 3390 if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX && 3391 (bp->flags & BNXT_FLAG_DIM) && bnxt_vnic_is_active(bp)) { 3392 struct dim_sample dim_sample = {}; 3393 3394 dim_update_sample(cpr->event_ctr, 3395 cpr_rx->rx_packets, 3396 cpr_rx->rx_bytes, 3397 &dim_sample); 3398 net_dim(&cpr->dim, &dim_sample); 3399 } 3400 return work_done; 3401} 3402 3403static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, 3404 struct bnxt_tx_ring_info *txr, int idx) 3405{ 3406 int i, max_idx; 3407 struct pci_dev *pdev = bp->pdev; 3408 3409 max_idx = bp->tx_nr_pages * TX_DESC_CNT; 3410 3411 for (i = 0; i < max_idx;) { 3412 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[i]; 3413 struct sk_buff *skb; 3414 int j, last; 3415 3416 if (idx < bp->tx_nr_rings_xdp && 3417 tx_buf->action == XDP_REDIRECT) { 3418 dma_unmap_single(&pdev->dev, 3419 dma_unmap_addr(tx_buf, mapping), 3420 dma_unmap_len(tx_buf, len), 3421 DMA_TO_DEVICE); 3422 xdp_return_frame(tx_buf->xdpf); 3423 tx_buf->action = 0; 3424 tx_buf->xdpf = NULL; 3425 i++; 3426 continue; 3427 } 3428 3429 skb = tx_buf->skb; 3430 if (!skb) { 3431 i++; 3432 continue; 3433 } 3434 3435 tx_buf->skb = NULL; 3436 3437 if (tx_buf->is_push) { 3438 dev_kfree_skb(skb); 3439 i += 2; 3440 continue; 3441 } 3442 3443 dma_unmap_single(&pdev->dev, 3444 dma_unmap_addr(tx_buf, mapping), 3445 skb_headlen(skb), 3446 DMA_TO_DEVICE); 3447 3448 last = tx_buf->nr_frags; 3449 i += 2; 3450 for (j = 0; j < last; j++, i++) { 3451 int ring_idx = i & bp->tx_ring_mask; 3452 skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; 3453 3454 tx_buf = &txr->tx_buf_ring[ring_idx]; 3455 netmem_dma_unmap_page_attrs(&pdev->dev, 3456 dma_unmap_addr(tx_buf, 3457 mapping), 3458 skb_frag_size(frag), 3459 DMA_TO_DEVICE, 0); 3460 } 3461 dev_kfree_skb(skb); 3462 } 3463 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, idx)); 3464} 3465 3466static void bnxt_free_tx_skbs(struct bnxt *bp) 3467{ 3468 int i; 3469 3470 if (!bp->tx_ring) 3471 return; 3472 3473 for (i = 0; i < bp->tx_nr_rings; i++) { 3474 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3475 3476 if (!txr->tx_buf_ring) 3477 continue; 3478 3479 bnxt_free_one_tx_ring_skbs(bp, txr, i); 3480 } 3481 3482 if (bp->ptp_cfg && !(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP)) 3483 bnxt_ptp_free_txts_skbs(bp->ptp_cfg); 3484} 3485 3486static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 3487{ 3488 int i, max_idx; 3489 3490 max_idx = bp->rx_nr_pages * RX_DESC_CNT; 3491 3492 for (i = 0; i < max_idx; i++) { 3493 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i]; 3494 void *data = rx_buf->data; 3495 3496 if (!data) 3497 continue; 3498 3499 rx_buf->data = NULL; 3500 if (BNXT_RX_PAGE_MODE(bp)) 3501 page_pool_recycle_direct(rxr->page_pool, data); 3502 else 3503 page_pool_free_va(rxr->head_pool, data, true); 3504 } 3505} 3506 3507static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 3508{ 3509 int i, max_idx; 3510 3511 max_idx = bp->rx_agg_nr_pages * RX_DESC_CNT; 3512 3513 for (i = 0; i < max_idx; i++) { 3514 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i]; 3515 netmem_ref netmem = rx_agg_buf->netmem; 3516 3517 if (!netmem) 3518 continue; 3519 3520 rx_agg_buf->netmem = 0; 3521 __clear_bit(i, rxr->rx_agg_bmap); 3522 3523 page_pool_recycle_direct_netmem(rxr->page_pool, netmem); 3524 } 3525} 3526 3527static void bnxt_free_one_tpa_info_data(struct bnxt *bp, 3528 struct bnxt_rx_ring_info *rxr) 3529{ 3530 int i; 3531 3532 for (i = 0; i < bp->max_tpa; i++) { 3533 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i]; 3534 u8 *data = tpa_info->data; 3535 3536 if (!data) 3537 continue; 3538 3539 tpa_info->data = NULL; 3540 page_pool_free_va(rxr->head_pool, data, false); 3541 } 3542} 3543 3544static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, 3545 struct bnxt_rx_ring_info *rxr) 3546{ 3547 struct bnxt_tpa_idx_map *map; 3548 3549 if (!rxr->rx_tpa) 3550 goto skip_rx_tpa_free; 3551 3552 bnxt_free_one_tpa_info_data(bp, rxr); 3553 3554skip_rx_tpa_free: 3555 if (!rxr->rx_buf_ring) 3556 goto skip_rx_buf_free; 3557 3558 bnxt_free_one_rx_ring(bp, rxr); 3559 3560skip_rx_buf_free: 3561 if (!rxr->rx_agg_ring) 3562 goto skip_rx_agg_free; 3563 3564 bnxt_free_one_rx_agg_ring(bp, rxr); 3565 3566skip_rx_agg_free: 3567 map = rxr->rx_tpa_idx_map; 3568 if (map) 3569 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap)); 3570} 3571 3572static void bnxt_free_rx_skbs(struct bnxt *bp) 3573{ 3574 int i; 3575 3576 if (!bp->rx_ring) 3577 return; 3578 3579 for (i = 0; i < bp->rx_nr_rings; i++) 3580 bnxt_free_one_rx_ring_skbs(bp, &bp->rx_ring[i]); 3581} 3582 3583static void bnxt_free_skbs(struct bnxt *bp) 3584{ 3585 bnxt_free_tx_skbs(bp); 3586 bnxt_free_rx_skbs(bp); 3587} 3588 3589static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len) 3590{ 3591 u8 init_val = ctxm->init_value; 3592 u16 offset = ctxm->init_offset; 3593 u8 *p2 = p; 3594 int i; 3595 3596 if (!init_val) 3597 return; 3598 if (offset == BNXT_CTX_INIT_INVALID_OFFSET) { 3599 memset(p, init_val, len); 3600 return; 3601 } 3602 for (i = 0; i < len; i += ctxm->entry_size) 3603 *(p2 + i + offset) = init_val; 3604} 3605 3606static size_t __bnxt_copy_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem, 3607 void *buf, size_t offset, size_t head, 3608 size_t tail) 3609{ 3610 int i, head_page, start_idx, source_offset; 3611 size_t len, rem_len, total_len, max_bytes; 3612 3613 head_page = head / rmem->page_size; 3614 source_offset = head % rmem->page_size; 3615 total_len = (tail - head) & MAX_CTX_BYTES_MASK; 3616 if (!total_len) 3617 total_len = MAX_CTX_BYTES; 3618 start_idx = head_page % MAX_CTX_PAGES; 3619 max_bytes = (rmem->nr_pages - start_idx) * rmem->page_size - 3620 source_offset; 3621 total_len = min(total_len, max_bytes); 3622 rem_len = total_len; 3623 3624 for (i = start_idx; rem_len; i++, source_offset = 0) { 3625 len = min((size_t)(rmem->page_size - source_offset), rem_len); 3626 if (buf) 3627 memcpy(buf + offset, rmem->pg_arr[i] + source_offset, 3628 len); 3629 offset += len; 3630 rem_len -= len; 3631 } 3632 return total_len; 3633} 3634 3635static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3636{ 3637 struct pci_dev *pdev = bp->pdev; 3638 int i; 3639 3640 if (!rmem->pg_arr) 3641 goto skip_pages; 3642 3643 for (i = 0; i < rmem->nr_pages; i++) { 3644 if (!rmem->pg_arr[i]) 3645 continue; 3646 3647 dma_free_coherent(&pdev->dev, rmem->page_size, 3648 rmem->pg_arr[i], rmem->dma_arr[i]); 3649 3650 rmem->pg_arr[i] = NULL; 3651 } 3652skip_pages: 3653 if (rmem->pg_tbl) { 3654 size_t pg_tbl_size = rmem->nr_pages * 8; 3655 3656 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3657 pg_tbl_size = rmem->page_size; 3658 dma_free_coherent(&pdev->dev, pg_tbl_size, 3659 rmem->pg_tbl, rmem->pg_tbl_map); 3660 rmem->pg_tbl = NULL; 3661 } 3662 if (rmem->vmem_size && *rmem->vmem) { 3663 vfree(*rmem->vmem); 3664 *rmem->vmem = NULL; 3665 } 3666} 3667 3668static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem) 3669{ 3670 struct pci_dev *pdev = bp->pdev; 3671 u64 valid_bit = 0; 3672 int i; 3673 3674 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) 3675 valid_bit = PTU_PTE_VALID; 3676 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) { 3677 size_t pg_tbl_size = rmem->nr_pages * 8; 3678 3679 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) 3680 pg_tbl_size = rmem->page_size; 3681 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size, 3682 &rmem->pg_tbl_map, 3683 GFP_KERNEL); 3684 if (!rmem->pg_tbl) 3685 return -ENOMEM; 3686 } 3687 3688 for (i = 0; i < rmem->nr_pages; i++) { 3689 u64 extra_bits = valid_bit; 3690 3691 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev, 3692 rmem->page_size, 3693 &rmem->dma_arr[i], 3694 GFP_KERNEL); 3695 if (!rmem->pg_arr[i]) 3696 return -ENOMEM; 3697 3698 if (rmem->ctx_mem) 3699 bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i], 3700 rmem->page_size); 3701 if (rmem->nr_pages > 1 || rmem->depth > 0) { 3702 if (i == rmem->nr_pages - 2 && 3703 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3704 extra_bits |= PTU_PTE_NEXT_TO_LAST; 3705 else if (i == rmem->nr_pages - 1 && 3706 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) 3707 extra_bits |= PTU_PTE_LAST; 3708 rmem->pg_tbl[i] = 3709 cpu_to_le64(rmem->dma_arr[i] | extra_bits); 3710 } 3711 } 3712 3713 if (rmem->vmem_size) { 3714 *rmem->vmem = vzalloc(rmem->vmem_size); 3715 if (!(*rmem->vmem)) 3716 return -ENOMEM; 3717 } 3718 return 0; 3719} 3720 3721static void bnxt_free_one_tpa_info(struct bnxt *bp, 3722 struct bnxt_rx_ring_info *rxr) 3723{ 3724 int i; 3725 3726 kfree(rxr->rx_tpa_idx_map); 3727 rxr->rx_tpa_idx_map = NULL; 3728 if (rxr->rx_tpa) { 3729 for (i = 0; i < bp->max_tpa; i++) { 3730 kfree(rxr->rx_tpa[i].agg_arr); 3731 rxr->rx_tpa[i].agg_arr = NULL; 3732 } 3733 } 3734 kfree(rxr->rx_tpa); 3735 rxr->rx_tpa = NULL; 3736} 3737 3738static void bnxt_free_tpa_info(struct bnxt *bp) 3739{ 3740 int i; 3741 3742 for (i = 0; i < bp->rx_nr_rings; i++) { 3743 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3744 3745 bnxt_free_one_tpa_info(bp, rxr); 3746 } 3747} 3748 3749static int bnxt_alloc_one_tpa_info(struct bnxt *bp, 3750 struct bnxt_rx_ring_info *rxr) 3751{ 3752 struct rx_agg_cmp *agg; 3753 int i; 3754 3755 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info), 3756 GFP_KERNEL); 3757 if (!rxr->rx_tpa) 3758 return -ENOMEM; 3759 3760 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 3761 return 0; 3762 for (i = 0; i < bp->max_tpa; i++) { 3763 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL); 3764 if (!agg) 3765 return -ENOMEM; 3766 rxr->rx_tpa[i].agg_arr = agg; 3767 } 3768 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map), 3769 GFP_KERNEL); 3770 if (!rxr->rx_tpa_idx_map) 3771 return -ENOMEM; 3772 3773 return 0; 3774} 3775 3776static int bnxt_alloc_tpa_info(struct bnxt *bp) 3777{ 3778 int i, rc; 3779 3780 bp->max_tpa = MAX_TPA; 3781 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 3782 if (!bp->max_tpa_v2) 3783 return 0; 3784 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5); 3785 } 3786 3787 for (i = 0; i < bp->rx_nr_rings; i++) { 3788 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3789 3790 rc = bnxt_alloc_one_tpa_info(bp, rxr); 3791 if (rc) 3792 return rc; 3793 } 3794 return 0; 3795} 3796 3797static void bnxt_free_rx_rings(struct bnxt *bp) 3798{ 3799 int i; 3800 3801 if (!bp->rx_ring) 3802 return; 3803 3804 bnxt_free_tpa_info(bp); 3805 for (i = 0; i < bp->rx_nr_rings; i++) { 3806 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3807 struct bnxt_ring_struct *ring; 3808 3809 if (rxr->xdp_prog) 3810 bpf_prog_put(rxr->xdp_prog); 3811 3812 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) 3813 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3814 3815 page_pool_destroy(rxr->page_pool); 3816 page_pool_destroy(rxr->head_pool); 3817 rxr->page_pool = rxr->head_pool = NULL; 3818 3819 kfree(rxr->rx_agg_bmap); 3820 rxr->rx_agg_bmap = NULL; 3821 3822 ring = &rxr->rx_ring_struct; 3823 bnxt_free_ring(bp, &ring->ring_mem); 3824 3825 ring = &rxr->rx_agg_ring_struct; 3826 bnxt_free_ring(bp, &ring->ring_mem); 3827 } 3828} 3829 3830static int bnxt_alloc_rx_page_pool(struct bnxt *bp, 3831 struct bnxt_rx_ring_info *rxr, 3832 int numa_node) 3833{ 3834 const unsigned int agg_size_fac = PAGE_SIZE / BNXT_RX_PAGE_SIZE; 3835 const unsigned int rx_size_fac = PAGE_SIZE / SZ_4K; 3836 struct page_pool_params pp = { 0 }; 3837 struct page_pool *pool; 3838 3839 pp.pool_size = bp->rx_agg_ring_size / agg_size_fac; 3840 if (BNXT_RX_PAGE_MODE(bp)) 3841 pp.pool_size += bp->rx_ring_size / rx_size_fac; 3842 pp.nid = numa_node; 3843 pp.netdev = bp->dev; 3844 pp.dev = &bp->pdev->dev; 3845 pp.dma_dir = bp->rx_dir; 3846 pp.max_len = PAGE_SIZE; 3847 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | 3848 PP_FLAG_ALLOW_UNREADABLE_NETMEM; 3849 pp.queue_idx = rxr->bnapi->index; 3850 3851 pool = page_pool_create(&pp); 3852 if (IS_ERR(pool)) 3853 return PTR_ERR(pool); 3854 rxr->page_pool = pool; 3855 3856 rxr->need_head_pool = page_pool_is_unreadable(pool); 3857 if (bnxt_separate_head_pool(rxr)) { 3858 pp.pool_size = min(bp->rx_ring_size / rx_size_fac, 1024); 3859 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 3860 pool = page_pool_create(&pp); 3861 if (IS_ERR(pool)) 3862 goto err_destroy_pp; 3863 } else { 3864 page_pool_get(pool); 3865 } 3866 rxr->head_pool = pool; 3867 3868 return 0; 3869 3870err_destroy_pp: 3871 page_pool_destroy(rxr->page_pool); 3872 rxr->page_pool = NULL; 3873 return PTR_ERR(pool); 3874} 3875 3876static void bnxt_enable_rx_page_pool(struct bnxt_rx_ring_info *rxr) 3877{ 3878 page_pool_enable_direct_recycling(rxr->head_pool, &rxr->bnapi->napi); 3879 page_pool_enable_direct_recycling(rxr->page_pool, &rxr->bnapi->napi); 3880} 3881 3882static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 3883{ 3884 u16 mem_size; 3885 3886 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1; 3887 mem_size = rxr->rx_agg_bmap_size / 8; 3888 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL); 3889 if (!rxr->rx_agg_bmap) 3890 return -ENOMEM; 3891 3892 return 0; 3893} 3894 3895static int bnxt_alloc_rx_rings(struct bnxt *bp) 3896{ 3897 int numa_node = dev_to_node(&bp->pdev->dev); 3898 int i, rc = 0, agg_rings = 0, cpu; 3899 3900 if (!bp->rx_ring) 3901 return -ENOMEM; 3902 3903 if (bp->flags & BNXT_FLAG_AGG_RINGS) 3904 agg_rings = 1; 3905 3906 for (i = 0; i < bp->rx_nr_rings; i++) { 3907 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 3908 struct bnxt_ring_struct *ring; 3909 int cpu_node; 3910 3911 ring = &rxr->rx_ring_struct; 3912 3913 cpu = cpumask_local_spread(i, numa_node); 3914 cpu_node = cpu_to_node(cpu); 3915 netdev_dbg(bp->dev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n", 3916 i, cpu_node); 3917 rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node); 3918 if (rc) 3919 return rc; 3920 bnxt_enable_rx_page_pool(rxr); 3921 3922 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); 3923 if (rc < 0) 3924 return rc; 3925 3926 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, 3927 MEM_TYPE_PAGE_POOL, 3928 rxr->page_pool); 3929 if (rc) { 3930 xdp_rxq_info_unreg(&rxr->xdp_rxq); 3931 return rc; 3932 } 3933 3934 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3935 if (rc) 3936 return rc; 3937 3938 ring->grp_idx = i; 3939 if (agg_rings) { 3940 ring = &rxr->rx_agg_ring_struct; 3941 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 3942 if (rc) 3943 return rc; 3944 3945 ring->grp_idx = i; 3946 rc = bnxt_alloc_rx_agg_bmap(bp, rxr); 3947 if (rc) 3948 return rc; 3949 } 3950 } 3951 if (bp->flags & BNXT_FLAG_TPA) 3952 rc = bnxt_alloc_tpa_info(bp); 3953 return rc; 3954} 3955 3956static void bnxt_free_tx_rings(struct bnxt *bp) 3957{ 3958 int i; 3959 struct pci_dev *pdev = bp->pdev; 3960 3961 if (!bp->tx_ring) 3962 return; 3963 3964 for (i = 0; i < bp->tx_nr_rings; i++) { 3965 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 3966 struct bnxt_ring_struct *ring; 3967 3968 if (txr->tx_push) { 3969 dma_free_coherent(&pdev->dev, bp->tx_push_size, 3970 txr->tx_push, txr->tx_push_mapping); 3971 txr->tx_push = NULL; 3972 } 3973 3974 ring = &txr->tx_ring_struct; 3975 3976 bnxt_free_ring(bp, &ring->ring_mem); 3977 } 3978} 3979 3980#define BNXT_TC_TO_RING_BASE(bp, tc) \ 3981 ((tc) * (bp)->tx_nr_rings_per_tc) 3982 3983#define BNXT_RING_TO_TC_OFF(bp, tx) \ 3984 ((tx) % (bp)->tx_nr_rings_per_tc) 3985 3986#define BNXT_RING_TO_TC(bp, tx) \ 3987 ((tx) / (bp)->tx_nr_rings_per_tc) 3988 3989static int bnxt_alloc_tx_rings(struct bnxt *bp) 3990{ 3991 int i, j, rc; 3992 struct pci_dev *pdev = bp->pdev; 3993 3994 bp->tx_push_size = 0; 3995 if (bp->tx_push_thresh) { 3996 int push_size; 3997 3998 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) + 3999 bp->tx_push_thresh); 4000 4001 if (push_size > 256) { 4002 push_size = 0; 4003 bp->tx_push_thresh = 0; 4004 } 4005 4006 bp->tx_push_size = push_size; 4007 } 4008 4009 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) { 4010 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4011 struct bnxt_ring_struct *ring; 4012 u8 qidx; 4013 4014 ring = &txr->tx_ring_struct; 4015 4016 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 4017 if (rc) 4018 return rc; 4019 4020 ring->grp_idx = txr->bnapi->index; 4021 if (bp->tx_push_size) { 4022 dma_addr_t mapping; 4023 4024 /* One pre-allocated DMA buffer to backup 4025 * TX push operation 4026 */ 4027 txr->tx_push = dma_alloc_coherent(&pdev->dev, 4028 bp->tx_push_size, 4029 &txr->tx_push_mapping, 4030 GFP_KERNEL); 4031 4032 if (!txr->tx_push) 4033 return -ENOMEM; 4034 4035 mapping = txr->tx_push_mapping + 4036 sizeof(struct tx_push_bd); 4037 txr->data_mapping = cpu_to_le64(mapping); 4038 } 4039 qidx = bp->tc_to_qidx[j]; 4040 ring->queue_id = bp->q_info[qidx].queue_id; 4041 spin_lock_init(&txr->xdp_tx_lock); 4042 if (i < bp->tx_nr_rings_xdp) 4043 continue; 4044 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1)) 4045 j++; 4046 } 4047 return 0; 4048} 4049 4050static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr) 4051{ 4052 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4053 4054 kfree(cpr->cp_desc_ring); 4055 cpr->cp_desc_ring = NULL; 4056 ring->ring_mem.pg_arr = NULL; 4057 kfree(cpr->cp_desc_mapping); 4058 cpr->cp_desc_mapping = NULL; 4059 ring->ring_mem.dma_arr = NULL; 4060} 4061 4062static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n) 4063{ 4064 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL); 4065 if (!cpr->cp_desc_ring) 4066 return -ENOMEM; 4067 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping), 4068 GFP_KERNEL); 4069 if (!cpr->cp_desc_mapping) 4070 return -ENOMEM; 4071 return 0; 4072} 4073 4074static void bnxt_free_all_cp_arrays(struct bnxt *bp) 4075{ 4076 int i; 4077 4078 if (!bp->bnapi) 4079 return; 4080 for (i = 0; i < bp->cp_nr_rings; i++) { 4081 struct bnxt_napi *bnapi = bp->bnapi[i]; 4082 4083 if (!bnapi) 4084 continue; 4085 bnxt_free_cp_arrays(&bnapi->cp_ring); 4086 } 4087} 4088 4089static int bnxt_alloc_all_cp_arrays(struct bnxt *bp) 4090{ 4091 int i, n = bp->cp_nr_pages; 4092 4093 for (i = 0; i < bp->cp_nr_rings; i++) { 4094 struct bnxt_napi *bnapi = bp->bnapi[i]; 4095 int rc; 4096 4097 if (!bnapi) 4098 continue; 4099 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n); 4100 if (rc) 4101 return rc; 4102 } 4103 return 0; 4104} 4105 4106static void bnxt_free_cp_rings(struct bnxt *bp) 4107{ 4108 int i; 4109 4110 if (!bp->bnapi) 4111 return; 4112 4113 for (i = 0; i < bp->cp_nr_rings; i++) { 4114 struct bnxt_napi *bnapi = bp->bnapi[i]; 4115 struct bnxt_cp_ring_info *cpr; 4116 struct bnxt_ring_struct *ring; 4117 int j; 4118 4119 if (!bnapi) 4120 continue; 4121 4122 cpr = &bnapi->cp_ring; 4123 ring = &cpr->cp_ring_struct; 4124 4125 bnxt_free_ring(bp, &ring->ring_mem); 4126 4127 if (!cpr->cp_ring_arr) 4128 continue; 4129 4130 for (j = 0; j < cpr->cp_ring_count; j++) { 4131 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 4132 4133 ring = &cpr2->cp_ring_struct; 4134 bnxt_free_ring(bp, &ring->ring_mem); 4135 bnxt_free_cp_arrays(cpr2); 4136 } 4137 kfree(cpr->cp_ring_arr); 4138 cpr->cp_ring_arr = NULL; 4139 cpr->cp_ring_count = 0; 4140 } 4141} 4142 4143static int bnxt_alloc_cp_sub_ring(struct bnxt *bp, 4144 struct bnxt_cp_ring_info *cpr) 4145{ 4146 struct bnxt_ring_mem_info *rmem; 4147 struct bnxt_ring_struct *ring; 4148 int rc; 4149 4150 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages); 4151 if (rc) { 4152 bnxt_free_cp_arrays(cpr); 4153 return -ENOMEM; 4154 } 4155 ring = &cpr->cp_ring_struct; 4156 rmem = &ring->ring_mem; 4157 rmem->nr_pages = bp->cp_nr_pages; 4158 rmem->page_size = HW_CMPD_RING_SIZE; 4159 rmem->pg_arr = (void **)cpr->cp_desc_ring; 4160 rmem->dma_arr = cpr->cp_desc_mapping; 4161 rmem->flags = BNXT_RMEM_RING_PTE_FLAG; 4162 rc = bnxt_alloc_ring(bp, rmem); 4163 if (rc) { 4164 bnxt_free_ring(bp, rmem); 4165 bnxt_free_cp_arrays(cpr); 4166 } 4167 return rc; 4168} 4169 4170static int bnxt_alloc_cp_rings(struct bnxt *bp) 4171{ 4172 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS); 4173 int i, j, rc, ulp_msix; 4174 int tcs = bp->num_tc; 4175 4176 if (!tcs) 4177 tcs = 1; 4178 ulp_msix = bnxt_get_ulp_msix_num(bp); 4179 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 4180 struct bnxt_napi *bnapi = bp->bnapi[i]; 4181 struct bnxt_cp_ring_info *cpr, *cpr2; 4182 struct bnxt_ring_struct *ring; 4183 int cp_count = 0, k; 4184 int rx = 0, tx = 0; 4185 4186 if (!bnapi) 4187 continue; 4188 4189 cpr = &bnapi->cp_ring; 4190 cpr->bnapi = bnapi; 4191 ring = &cpr->cp_ring_struct; 4192 4193 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 4194 if (rc) 4195 return rc; 4196 4197 ring->map_idx = ulp_msix + i; 4198 4199 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 4200 continue; 4201 4202 if (i < bp->rx_nr_rings) { 4203 cp_count++; 4204 rx = 1; 4205 } 4206 if (i < bp->tx_nr_rings_xdp) { 4207 cp_count++; 4208 tx = 1; 4209 } else if ((sh && i < bp->tx_nr_rings) || 4210 (!sh && i >= bp->rx_nr_rings)) { 4211 cp_count += tcs; 4212 tx = 1; 4213 } 4214 4215 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr), 4216 GFP_KERNEL); 4217 if (!cpr->cp_ring_arr) 4218 return -ENOMEM; 4219 cpr->cp_ring_count = cp_count; 4220 4221 for (k = 0; k < cp_count; k++) { 4222 cpr2 = &cpr->cp_ring_arr[k]; 4223 rc = bnxt_alloc_cp_sub_ring(bp, cpr2); 4224 if (rc) 4225 return rc; 4226 cpr2->bnapi = bnapi; 4227 cpr2->sw_stats = cpr->sw_stats; 4228 cpr2->cp_idx = k; 4229 if (!k && rx) { 4230 bp->rx_ring[i].rx_cpr = cpr2; 4231 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX; 4232 } else { 4233 int n, tc = k - rx; 4234 4235 n = BNXT_TC_TO_RING_BASE(bp, tc) + j; 4236 bp->tx_ring[n].tx_cpr = cpr2; 4237 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX; 4238 } 4239 } 4240 if (tx) 4241 j++; 4242 } 4243 return 0; 4244} 4245 4246static void bnxt_init_rx_ring_struct(struct bnxt *bp, 4247 struct bnxt_rx_ring_info *rxr) 4248{ 4249 struct bnxt_ring_mem_info *rmem; 4250 struct bnxt_ring_struct *ring; 4251 4252 ring = &rxr->rx_ring_struct; 4253 rmem = &ring->ring_mem; 4254 rmem->nr_pages = bp->rx_nr_pages; 4255 rmem->page_size = HW_RXBD_RING_SIZE; 4256 rmem->pg_arr = (void **)rxr->rx_desc_ring; 4257 rmem->dma_arr = rxr->rx_desc_mapping; 4258 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 4259 rmem->vmem = (void **)&rxr->rx_buf_ring; 4260 4261 ring = &rxr->rx_agg_ring_struct; 4262 rmem = &ring->ring_mem; 4263 rmem->nr_pages = bp->rx_agg_nr_pages; 4264 rmem->page_size = HW_RXBD_RING_SIZE; 4265 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 4266 rmem->dma_arr = rxr->rx_agg_desc_mapping; 4267 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 4268 rmem->vmem = (void **)&rxr->rx_agg_ring; 4269} 4270 4271static void bnxt_reset_rx_ring_struct(struct bnxt *bp, 4272 struct bnxt_rx_ring_info *rxr) 4273{ 4274 struct bnxt_ring_mem_info *rmem; 4275 struct bnxt_ring_struct *ring; 4276 int i; 4277 4278 rxr->page_pool->p.napi = NULL; 4279 rxr->page_pool = NULL; 4280 rxr->head_pool->p.napi = NULL; 4281 rxr->head_pool = NULL; 4282 memset(&rxr->xdp_rxq, 0, sizeof(struct xdp_rxq_info)); 4283 4284 ring = &rxr->rx_ring_struct; 4285 rmem = &ring->ring_mem; 4286 rmem->pg_tbl = NULL; 4287 rmem->pg_tbl_map = 0; 4288 for (i = 0; i < rmem->nr_pages; i++) { 4289 rmem->pg_arr[i] = NULL; 4290 rmem->dma_arr[i] = 0; 4291 } 4292 *rmem->vmem = NULL; 4293 4294 ring = &rxr->rx_agg_ring_struct; 4295 rmem = &ring->ring_mem; 4296 rmem->pg_tbl = NULL; 4297 rmem->pg_tbl_map = 0; 4298 for (i = 0; i < rmem->nr_pages; i++) { 4299 rmem->pg_arr[i] = NULL; 4300 rmem->dma_arr[i] = 0; 4301 } 4302 *rmem->vmem = NULL; 4303} 4304 4305static void bnxt_init_ring_struct(struct bnxt *bp) 4306{ 4307 int i, j; 4308 4309 for (i = 0; i < bp->cp_nr_rings; i++) { 4310 struct bnxt_napi *bnapi = bp->bnapi[i]; 4311 struct bnxt_ring_mem_info *rmem; 4312 struct bnxt_cp_ring_info *cpr; 4313 struct bnxt_rx_ring_info *rxr; 4314 struct bnxt_tx_ring_info *txr; 4315 struct bnxt_ring_struct *ring; 4316 4317 if (!bnapi) 4318 continue; 4319 4320 cpr = &bnapi->cp_ring; 4321 ring = &cpr->cp_ring_struct; 4322 rmem = &ring->ring_mem; 4323 rmem->nr_pages = bp->cp_nr_pages; 4324 rmem->page_size = HW_CMPD_RING_SIZE; 4325 rmem->pg_arr = (void **)cpr->cp_desc_ring; 4326 rmem->dma_arr = cpr->cp_desc_mapping; 4327 rmem->vmem_size = 0; 4328 4329 rxr = bnapi->rx_ring; 4330 if (!rxr) 4331 goto skip_rx; 4332 4333 ring = &rxr->rx_ring_struct; 4334 rmem = &ring->ring_mem; 4335 rmem->nr_pages = bp->rx_nr_pages; 4336 rmem->page_size = HW_RXBD_RING_SIZE; 4337 rmem->pg_arr = (void **)rxr->rx_desc_ring; 4338 rmem->dma_arr = rxr->rx_desc_mapping; 4339 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages; 4340 rmem->vmem = (void **)&rxr->rx_buf_ring; 4341 4342 ring = &rxr->rx_agg_ring_struct; 4343 rmem = &ring->ring_mem; 4344 rmem->nr_pages = bp->rx_agg_nr_pages; 4345 rmem->page_size = HW_RXBD_RING_SIZE; 4346 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring; 4347 rmem->dma_arr = rxr->rx_agg_desc_mapping; 4348 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages; 4349 rmem->vmem = (void **)&rxr->rx_agg_ring; 4350 4351skip_rx: 4352 bnxt_for_each_napi_tx(j, bnapi, txr) { 4353 ring = &txr->tx_ring_struct; 4354 rmem = &ring->ring_mem; 4355 rmem->nr_pages = bp->tx_nr_pages; 4356 rmem->page_size = HW_TXBD_RING_SIZE; 4357 rmem->pg_arr = (void **)txr->tx_desc_ring; 4358 rmem->dma_arr = txr->tx_desc_mapping; 4359 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages; 4360 rmem->vmem = (void **)&txr->tx_buf_ring; 4361 } 4362 } 4363} 4364 4365static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type) 4366{ 4367 int i; 4368 u32 prod; 4369 struct rx_bd **rx_buf_ring; 4370 4371 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr; 4372 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) { 4373 int j; 4374 struct rx_bd *rxbd; 4375 4376 rxbd = rx_buf_ring[i]; 4377 if (!rxbd) 4378 continue; 4379 4380 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) { 4381 rxbd->rx_bd_len_flags_type = cpu_to_le32(type); 4382 rxbd->rx_bd_opaque = prod; 4383 } 4384 } 4385} 4386 4387static void bnxt_alloc_one_rx_ring_skb(struct bnxt *bp, 4388 struct bnxt_rx_ring_info *rxr, 4389 int ring_nr) 4390{ 4391 u32 prod; 4392 int i; 4393 4394 prod = rxr->rx_prod; 4395 for (i = 0; i < bp->rx_ring_size; i++) { 4396 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) { 4397 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d skbs only\n", 4398 ring_nr, i, bp->rx_ring_size); 4399 break; 4400 } 4401 prod = NEXT_RX(prod); 4402 } 4403 rxr->rx_prod = prod; 4404} 4405 4406static void bnxt_alloc_one_rx_ring_netmem(struct bnxt *bp, 4407 struct bnxt_rx_ring_info *rxr, 4408 int ring_nr) 4409{ 4410 u32 prod; 4411 int i; 4412 4413 prod = rxr->rx_agg_prod; 4414 for (i = 0; i < bp->rx_agg_ring_size; i++) { 4415 if (bnxt_alloc_rx_netmem(bp, rxr, prod, GFP_KERNEL)) { 4416 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d pages only\n", 4417 ring_nr, i, bp->rx_agg_ring_size); 4418 break; 4419 } 4420 prod = NEXT_RX_AGG(prod); 4421 } 4422 rxr->rx_agg_prod = prod; 4423} 4424 4425static int bnxt_alloc_one_tpa_info_data(struct bnxt *bp, 4426 struct bnxt_rx_ring_info *rxr) 4427{ 4428 dma_addr_t mapping; 4429 u8 *data; 4430 int i; 4431 4432 for (i = 0; i < bp->max_tpa; i++) { 4433 data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, 4434 GFP_KERNEL); 4435 if (!data) 4436 return -ENOMEM; 4437 4438 rxr->rx_tpa[i].data = data; 4439 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset; 4440 rxr->rx_tpa[i].mapping = mapping; 4441 } 4442 4443 return 0; 4444} 4445 4446static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr) 4447{ 4448 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 4449 int rc; 4450 4451 bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr); 4452 4453 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 4454 return 0; 4455 4456 bnxt_alloc_one_rx_ring_netmem(bp, rxr, ring_nr); 4457 4458 if (rxr->rx_tpa) { 4459 rc = bnxt_alloc_one_tpa_info_data(bp, rxr); 4460 if (rc) 4461 return rc; 4462 } 4463 return 0; 4464} 4465 4466static void bnxt_init_one_rx_ring_rxbd(struct bnxt *bp, 4467 struct bnxt_rx_ring_info *rxr) 4468{ 4469 struct bnxt_ring_struct *ring; 4470 u32 type; 4471 4472 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) | 4473 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP; 4474 4475 if (NET_IP_ALIGN == 2) 4476 type |= RX_BD_FLAGS_SOP; 4477 4478 ring = &rxr->rx_ring_struct; 4479 bnxt_init_rxbd_pages(ring, type); 4480 ring->fw_ring_id = INVALID_HW_RING_ID; 4481} 4482 4483static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp, 4484 struct bnxt_rx_ring_info *rxr) 4485{ 4486 struct bnxt_ring_struct *ring; 4487 u32 type; 4488 4489 ring = &rxr->rx_agg_ring_struct; 4490 ring->fw_ring_id = INVALID_HW_RING_ID; 4491 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { 4492 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | 4493 RX_BD_TYPE_RX_AGG_BD; 4494 4495 /* On P7, setting EOP will cause the chip to disable 4496 * Relaxed Ordering (RO) for TPA data. Disable EOP for 4497 * potentially higher performance with RO. 4498 */ 4499 if (BNXT_CHIP_P5_AND_MINUS(bp) || !(bp->flags & BNXT_FLAG_TPA)) 4500 type |= RX_BD_FLAGS_AGG_EOP; 4501 4502 bnxt_init_rxbd_pages(ring, type); 4503 } 4504} 4505 4506static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr) 4507{ 4508 struct bnxt_rx_ring_info *rxr; 4509 4510 rxr = &bp->rx_ring[ring_nr]; 4511 bnxt_init_one_rx_ring_rxbd(bp, rxr); 4512 4513 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX, 4514 &rxr->bnapi->napi); 4515 4516 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) { 4517 bpf_prog_add(bp->xdp_prog, 1); 4518 rxr->xdp_prog = bp->xdp_prog; 4519 } 4520 4521 bnxt_init_one_rx_agg_ring_rxbd(bp, rxr); 4522 4523 return bnxt_alloc_one_rx_ring(bp, ring_nr); 4524} 4525 4526static void bnxt_init_cp_rings(struct bnxt *bp) 4527{ 4528 int i, j; 4529 4530 for (i = 0; i < bp->cp_nr_rings; i++) { 4531 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; 4532 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 4533 4534 ring->fw_ring_id = INVALID_HW_RING_ID; 4535 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 4536 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 4537 if (!cpr->cp_ring_arr) 4538 continue; 4539 for (j = 0; j < cpr->cp_ring_count; j++) { 4540 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 4541 4542 ring = &cpr2->cp_ring_struct; 4543 ring->fw_ring_id = INVALID_HW_RING_ID; 4544 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; 4545 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; 4546 } 4547 } 4548} 4549 4550static int bnxt_init_rx_rings(struct bnxt *bp) 4551{ 4552 int i, rc = 0; 4553 4554 if (BNXT_RX_PAGE_MODE(bp)) { 4555 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; 4556 bp->rx_dma_offset = XDP_PACKET_HEADROOM; 4557 } else { 4558 bp->rx_offset = BNXT_RX_OFFSET; 4559 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; 4560 } 4561 4562 for (i = 0; i < bp->rx_nr_rings; i++) { 4563 rc = bnxt_init_one_rx_ring(bp, i); 4564 if (rc) 4565 break; 4566 } 4567 4568 return rc; 4569} 4570 4571static int bnxt_init_tx_rings(struct bnxt *bp) 4572{ 4573 u16 i; 4574 4575 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, 4576 BNXT_MIN_TX_DESC_CNT); 4577 4578 for (i = 0; i < bp->tx_nr_rings; i++) { 4579 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 4580 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 4581 4582 ring->fw_ring_id = INVALID_HW_RING_ID; 4583 4584 if (i >= bp->tx_nr_rings_xdp) 4585 netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp, 4586 NETDEV_QUEUE_TYPE_TX, 4587 &txr->bnapi->napi); 4588 } 4589 4590 return 0; 4591} 4592 4593static void bnxt_free_ring_grps(struct bnxt *bp) 4594{ 4595 kfree(bp->grp_info); 4596 bp->grp_info = NULL; 4597} 4598 4599static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init) 4600{ 4601 int i; 4602 4603 if (irq_re_init) { 4604 bp->grp_info = kcalloc(bp->cp_nr_rings, 4605 sizeof(struct bnxt_ring_grp_info), 4606 GFP_KERNEL); 4607 if (!bp->grp_info) 4608 return -ENOMEM; 4609 } 4610 for (i = 0; i < bp->cp_nr_rings; i++) { 4611 if (irq_re_init) 4612 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID; 4613 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 4614 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID; 4615 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID; 4616 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 4617 } 4618 return 0; 4619} 4620 4621static void bnxt_free_vnics(struct bnxt *bp) 4622{ 4623 kfree(bp->vnic_info); 4624 bp->vnic_info = NULL; 4625 bp->nr_vnics = 0; 4626} 4627 4628static int bnxt_alloc_vnics(struct bnxt *bp) 4629{ 4630 int num_vnics = 1; 4631 4632#ifdef CONFIG_RFS_ACCEL 4633 if (bp->flags & BNXT_FLAG_RFS) { 4634 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 4635 num_vnics++; 4636 else if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 4637 num_vnics += bp->rx_nr_rings; 4638 } 4639#endif 4640 4641 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 4642 num_vnics++; 4643 4644 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info), 4645 GFP_KERNEL); 4646 if (!bp->vnic_info) 4647 return -ENOMEM; 4648 4649 bp->nr_vnics = num_vnics; 4650 return 0; 4651} 4652 4653static void bnxt_init_vnics(struct bnxt *bp) 4654{ 4655 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 4656 int i; 4657 4658 for (i = 0; i < bp->nr_vnics; i++) { 4659 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 4660 int j; 4661 4662 vnic->fw_vnic_id = INVALID_HW_RING_ID; 4663 vnic->vnic_id = i; 4664 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) 4665 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID; 4666 4667 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID; 4668 4669 if (bp->vnic_info[i].rss_hash_key) { 4670 if (i == BNXT_VNIC_DEFAULT) { 4671 u8 *key = (void *)vnic->rss_hash_key; 4672 int k; 4673 4674 if (!bp->rss_hash_key_valid && 4675 !bp->rss_hash_key_updated) { 4676 get_random_bytes(bp->rss_hash_key, 4677 HW_HASH_KEY_SIZE); 4678 bp->rss_hash_key_updated = true; 4679 } 4680 4681 memcpy(vnic->rss_hash_key, bp->rss_hash_key, 4682 HW_HASH_KEY_SIZE); 4683 4684 if (!bp->rss_hash_key_updated) 4685 continue; 4686 4687 bp->rss_hash_key_updated = false; 4688 bp->rss_hash_key_valid = true; 4689 4690 bp->toeplitz_prefix = 0; 4691 for (k = 0; k < 8; k++) { 4692 bp->toeplitz_prefix <<= 8; 4693 bp->toeplitz_prefix |= key[k]; 4694 } 4695 } else { 4696 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key, 4697 HW_HASH_KEY_SIZE); 4698 } 4699 } 4700 } 4701} 4702 4703static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg) 4704{ 4705 int pages; 4706 4707 pages = ring_size / desc_per_pg; 4708 4709 if (!pages) 4710 return 1; 4711 4712 pages++; 4713 4714 while (pages & (pages - 1)) 4715 pages++; 4716 4717 return pages; 4718} 4719 4720void bnxt_set_tpa_flags(struct bnxt *bp) 4721{ 4722 bp->flags &= ~BNXT_FLAG_TPA; 4723 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 4724 return; 4725 if (bp->dev->features & NETIF_F_LRO) 4726 bp->flags |= BNXT_FLAG_LRO; 4727 else if (bp->dev->features & NETIF_F_GRO_HW) 4728 bp->flags |= BNXT_FLAG_GRO; 4729} 4730 4731static void bnxt_init_ring_params(struct bnxt *bp) 4732{ 4733 unsigned int rx_size; 4734 4735 bp->rx_copybreak = BNXT_DEFAULT_RX_COPYBREAK; 4736 /* Try to fit 4 chunks into a 4k page */ 4737 rx_size = SZ_1K - 4738 NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4739 bp->dev->cfg->hds_thresh = max(BNXT_DEFAULT_RX_COPYBREAK, rx_size); 4740} 4741 4742/* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must 4743 * be set on entry. 4744 */ 4745void bnxt_set_ring_params(struct bnxt *bp) 4746{ 4747 u32 ring_size, rx_size, rx_space, max_rx_cmpl; 4748 u32 agg_factor = 0, agg_ring_size = 0; 4749 4750 /* 8 for CRC and VLAN */ 4751 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8); 4752 4753 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) + 4754 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4755 4756 ring_size = bp->rx_ring_size; 4757 bp->rx_agg_ring_size = 0; 4758 bp->rx_agg_nr_pages = 0; 4759 4760 if (bp->flags & BNXT_FLAG_TPA || bp->flags & BNXT_FLAG_HDS) 4761 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE); 4762 4763 bp->flags &= ~BNXT_FLAG_JUMBO; 4764 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) { 4765 u32 jumbo_factor; 4766 4767 bp->flags |= BNXT_FLAG_JUMBO; 4768 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT; 4769 if (jumbo_factor > agg_factor) 4770 agg_factor = jumbo_factor; 4771 } 4772 if (agg_factor) { 4773 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) { 4774 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA; 4775 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n", 4776 bp->rx_ring_size, ring_size); 4777 bp->rx_ring_size = ring_size; 4778 } 4779 agg_ring_size = ring_size * agg_factor; 4780 4781 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size, 4782 RX_DESC_CNT); 4783 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) { 4784 u32 tmp = agg_ring_size; 4785 4786 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES; 4787 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1; 4788 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n", 4789 tmp, agg_ring_size); 4790 } 4791 bp->rx_agg_ring_size = agg_ring_size; 4792 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1; 4793 4794 if (BNXT_RX_PAGE_MODE(bp)) { 4795 rx_space = PAGE_SIZE; 4796 rx_size = PAGE_SIZE - 4797 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) - 4798 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4799 } else { 4800 rx_size = max3(BNXT_DEFAULT_RX_COPYBREAK, 4801 bp->rx_copybreak, 4802 bp->dev->cfg_pending->hds_thresh); 4803 rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN); 4804 rx_space = rx_size + NET_SKB_PAD + 4805 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 4806 } 4807 } 4808 4809 bp->rx_buf_use_size = rx_size; 4810 bp->rx_buf_size = rx_space; 4811 4812 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT); 4813 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1; 4814 4815 ring_size = bp->tx_ring_size; 4816 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT); 4817 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1; 4818 4819 max_rx_cmpl = bp->rx_ring_size; 4820 /* MAX TPA needs to be added because TPA_START completions are 4821 * immediately recycled, so the TPA completions are not bound by 4822 * the RX ring size. 4823 */ 4824 if (bp->flags & BNXT_FLAG_TPA) 4825 max_rx_cmpl += bp->max_tpa; 4826 /* RX and TPA completions are 32-byte, all others are 16-byte */ 4827 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size; 4828 bp->cp_ring_size = ring_size; 4829 4830 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT); 4831 if (bp->cp_nr_pages > MAX_CP_PAGES) { 4832 bp->cp_nr_pages = MAX_CP_PAGES; 4833 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1; 4834 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n", 4835 ring_size, bp->cp_ring_size); 4836 } 4837 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT; 4838 bp->cp_ring_mask = bp->cp_bit - 1; 4839} 4840 4841/* Changing allocation mode of RX rings. 4842 * TODO: Update when extending xdp_rxq_info to support allocation modes. 4843 */ 4844static void __bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 4845{ 4846 struct net_device *dev = bp->dev; 4847 4848 if (page_mode) { 4849 bp->flags &= ~(BNXT_FLAG_AGG_RINGS | BNXT_FLAG_NO_AGG_RINGS); 4850 bp->flags |= BNXT_FLAG_RX_PAGE_MODE; 4851 4852 if (bp->xdp_prog->aux->xdp_has_frags) 4853 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU); 4854 else 4855 dev->max_mtu = 4856 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); 4857 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { 4858 bp->flags |= BNXT_FLAG_JUMBO; 4859 bp->rx_skb_func = bnxt_rx_multi_page_skb; 4860 } else { 4861 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 4862 bp->rx_skb_func = bnxt_rx_page_skb; 4863 } 4864 bp->rx_dir = DMA_BIDIRECTIONAL; 4865 } else { 4866 dev->max_mtu = bp->max_mtu; 4867 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; 4868 bp->rx_dir = DMA_FROM_DEVICE; 4869 bp->rx_skb_func = bnxt_rx_skb; 4870 } 4871} 4872 4873void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) 4874{ 4875 __bnxt_set_rx_skb_mode(bp, page_mode); 4876 4877 if (!page_mode) { 4878 int rx, tx; 4879 4880 bnxt_get_max_rings(bp, &rx, &tx, true); 4881 if (rx > 1) { 4882 bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS; 4883 bp->dev->hw_features |= NETIF_F_LRO; 4884 } 4885 } 4886 4887 /* Update LRO and GRO_HW availability */ 4888 netdev_update_features(bp->dev); 4889} 4890 4891static void bnxt_free_vnic_attributes(struct bnxt *bp) 4892{ 4893 int i; 4894 struct bnxt_vnic_info *vnic; 4895 struct pci_dev *pdev = bp->pdev; 4896 4897 if (!bp->vnic_info) 4898 return; 4899 4900 for (i = 0; i < bp->nr_vnics; i++) { 4901 vnic = &bp->vnic_info[i]; 4902 4903 kfree(vnic->fw_grp_ids); 4904 vnic->fw_grp_ids = NULL; 4905 4906 kfree(vnic->uc_list); 4907 vnic->uc_list = NULL; 4908 4909 if (vnic->mc_list) { 4910 dma_free_coherent(&pdev->dev, vnic->mc_list_size, 4911 vnic->mc_list, vnic->mc_list_mapping); 4912 vnic->mc_list = NULL; 4913 } 4914 4915 if (vnic->rss_table) { 4916 dma_free_coherent(&pdev->dev, vnic->rss_table_size, 4917 vnic->rss_table, 4918 vnic->rss_table_dma_addr); 4919 vnic->rss_table = NULL; 4920 } 4921 4922 vnic->rss_hash_key = NULL; 4923 vnic->flags = 0; 4924 } 4925} 4926 4927static int bnxt_alloc_vnic_attributes(struct bnxt *bp) 4928{ 4929 int i, rc = 0, size; 4930 struct bnxt_vnic_info *vnic; 4931 struct pci_dev *pdev = bp->pdev; 4932 int max_rings; 4933 4934 for (i = 0; i < bp->nr_vnics; i++) { 4935 vnic = &bp->vnic_info[i]; 4936 4937 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) { 4938 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN; 4939 4940 if (mem_size > 0) { 4941 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL); 4942 if (!vnic->uc_list) { 4943 rc = -ENOMEM; 4944 goto out; 4945 } 4946 } 4947 } 4948 4949 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) { 4950 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN; 4951 vnic->mc_list = 4952 dma_alloc_coherent(&pdev->dev, 4953 vnic->mc_list_size, 4954 &vnic->mc_list_mapping, 4955 GFP_KERNEL); 4956 if (!vnic->mc_list) { 4957 rc = -ENOMEM; 4958 goto out; 4959 } 4960 } 4961 4962 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 4963 goto vnic_skip_grps; 4964 4965 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 4966 max_rings = bp->rx_nr_rings; 4967 else 4968 max_rings = 1; 4969 4970 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL); 4971 if (!vnic->fw_grp_ids) { 4972 rc = -ENOMEM; 4973 goto out; 4974 } 4975vnic_skip_grps: 4976 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && 4977 !(vnic->flags & BNXT_VNIC_RSS_FLAG)) 4978 continue; 4979 4980 /* Allocate rss table and hash key */ 4981 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16)); 4982 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 4983 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5); 4984 4985 vnic->rss_table_size = size + HW_HASH_KEY_SIZE; 4986 vnic->rss_table = dma_alloc_coherent(&pdev->dev, 4987 vnic->rss_table_size, 4988 &vnic->rss_table_dma_addr, 4989 GFP_KERNEL); 4990 if (!vnic->rss_table) { 4991 rc = -ENOMEM; 4992 goto out; 4993 } 4994 4995 vnic->rss_hash_key = ((void *)vnic->rss_table) + size; 4996 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size; 4997 } 4998 return 0; 4999 5000out: 5001 return rc; 5002} 5003 5004static void bnxt_free_hwrm_resources(struct bnxt *bp) 5005{ 5006 struct bnxt_hwrm_wait_token *token; 5007 5008 dma_pool_destroy(bp->hwrm_dma_pool); 5009 bp->hwrm_dma_pool = NULL; 5010 5011 rcu_read_lock(); 5012 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node) 5013 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED); 5014 rcu_read_unlock(); 5015} 5016 5017static int bnxt_alloc_hwrm_resources(struct bnxt *bp) 5018{ 5019 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev, 5020 BNXT_HWRM_DMA_SIZE, 5021 BNXT_HWRM_DMA_ALIGN, 0); 5022 if (!bp->hwrm_dma_pool) 5023 return -ENOMEM; 5024 5025 INIT_HLIST_HEAD(&bp->hwrm_pending_list); 5026 5027 return 0; 5028} 5029 5030static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats) 5031{ 5032 kfree(stats->hw_masks); 5033 stats->hw_masks = NULL; 5034 kfree(stats->sw_stats); 5035 stats->sw_stats = NULL; 5036 if (stats->hw_stats) { 5037 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats, 5038 stats->hw_stats_map); 5039 stats->hw_stats = NULL; 5040 } 5041} 5042 5043static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats, 5044 bool alloc_masks) 5045{ 5046 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len, 5047 &stats->hw_stats_map, GFP_KERNEL); 5048 if (!stats->hw_stats) 5049 return -ENOMEM; 5050 5051 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); 5052 if (!stats->sw_stats) 5053 goto stats_mem_err; 5054 5055 if (alloc_masks) { 5056 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); 5057 if (!stats->hw_masks) 5058 goto stats_mem_err; 5059 } 5060 return 0; 5061 5062stats_mem_err: 5063 bnxt_free_stats_mem(bp, stats); 5064 return -ENOMEM; 5065} 5066 5067static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count) 5068{ 5069 int i; 5070 5071 for (i = 0; i < count; i++) 5072 mask_arr[i] = mask; 5073} 5074 5075static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) 5076{ 5077 int i; 5078 5079 for (i = 0; i < count; i++) 5080 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); 5081} 5082 5083static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp, 5084 struct bnxt_stats_mem *stats) 5085{ 5086 struct hwrm_func_qstats_ext_output *resp; 5087 struct hwrm_func_qstats_ext_input *req; 5088 __le64 *hw_masks; 5089 int rc; 5090 5091 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) || 5092 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 5093 return -EOPNOTSUPP; 5094 5095 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT); 5096 if (rc) 5097 return rc; 5098 5099 req->fid = cpu_to_le16(0xffff); 5100 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 5101 5102 resp = hwrm_req_hold(bp, req); 5103 rc = hwrm_req_send(bp, req); 5104 if (!rc) { 5105 hw_masks = &resp->rx_ucast_pkts; 5106 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); 5107 } 5108 hwrm_req_drop(bp, req); 5109 return rc; 5110} 5111 5112static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags); 5113static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags); 5114 5115static void bnxt_init_stats(struct bnxt *bp) 5116{ 5117 struct bnxt_napi *bnapi = bp->bnapi[0]; 5118 struct bnxt_cp_ring_info *cpr; 5119 struct bnxt_stats_mem *stats; 5120 __le64 *rx_stats, *tx_stats; 5121 int rc, rx_count, tx_count; 5122 u64 *rx_masks, *tx_masks; 5123 u64 mask; 5124 u8 flags; 5125 5126 cpr = &bnapi->cp_ring; 5127 stats = &cpr->stats; 5128 rc = bnxt_hwrm_func_qstat_ext(bp, stats); 5129 if (rc) { 5130 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 5131 mask = (1ULL << 48) - 1; 5132 else 5133 mask = -1ULL; 5134 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8); 5135 } 5136 if (bp->flags & BNXT_FLAG_PORT_STATS) { 5137 stats = &bp->port_stats; 5138 rx_stats = stats->hw_stats; 5139 rx_masks = stats->hw_masks; 5140 rx_count = sizeof(struct rx_port_stats) / 8; 5141 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 5142 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 5143 tx_count = sizeof(struct tx_port_stats) / 8; 5144 5145 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; 5146 rc = bnxt_hwrm_port_qstats(bp, flags); 5147 if (rc) { 5148 mask = (1ULL << 40) - 1; 5149 5150 bnxt_fill_masks(rx_masks, mask, rx_count); 5151 bnxt_fill_masks(tx_masks, mask, tx_count); 5152 } else { 5153 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 5154 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count); 5155 bnxt_hwrm_port_qstats(bp, 0); 5156 } 5157 } 5158 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 5159 stats = &bp->rx_port_stats_ext; 5160 rx_stats = stats->hw_stats; 5161 rx_masks = stats->hw_masks; 5162 rx_count = sizeof(struct rx_port_stats_ext) / 8; 5163 stats = &bp->tx_port_stats_ext; 5164 tx_stats = stats->hw_stats; 5165 tx_masks = stats->hw_masks; 5166 tx_count = sizeof(struct tx_port_stats_ext) / 8; 5167 5168 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; 5169 rc = bnxt_hwrm_port_qstats_ext(bp, flags); 5170 if (rc) { 5171 mask = (1ULL << 40) - 1; 5172 5173 bnxt_fill_masks(rx_masks, mask, rx_count); 5174 if (tx_stats) 5175 bnxt_fill_masks(tx_masks, mask, tx_count); 5176 } else { 5177 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count); 5178 if (tx_stats) 5179 bnxt_copy_hw_masks(tx_masks, tx_stats, 5180 tx_count); 5181 bnxt_hwrm_port_qstats_ext(bp, 0); 5182 } 5183 } 5184} 5185 5186static void bnxt_free_port_stats(struct bnxt *bp) 5187{ 5188 bp->flags &= ~BNXT_FLAG_PORT_STATS; 5189 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT; 5190 5191 bnxt_free_stats_mem(bp, &bp->port_stats); 5192 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext); 5193 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext); 5194} 5195 5196static void bnxt_free_ring_stats(struct bnxt *bp) 5197{ 5198 int i; 5199 5200 if (!bp->bnapi) 5201 return; 5202 5203 for (i = 0; i < bp->cp_nr_rings; i++) { 5204 struct bnxt_napi *bnapi = bp->bnapi[i]; 5205 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5206 5207 bnxt_free_stats_mem(bp, &cpr->stats); 5208 5209 kfree(cpr->sw_stats); 5210 cpr->sw_stats = NULL; 5211 } 5212} 5213 5214static int bnxt_alloc_stats(struct bnxt *bp) 5215{ 5216 u32 size, i; 5217 int rc; 5218 5219 size = bp->hw_ring_stats_size; 5220 5221 for (i = 0; i < bp->cp_nr_rings; i++) { 5222 struct bnxt_napi *bnapi = bp->bnapi[i]; 5223 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5224 5225 cpr->sw_stats = kzalloc(sizeof(*cpr->sw_stats), GFP_KERNEL); 5226 if (!cpr->sw_stats) 5227 return -ENOMEM; 5228 5229 cpr->stats.len = size; 5230 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i); 5231 if (rc) 5232 return rc; 5233 5234 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 5235 } 5236 5237 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700) 5238 return 0; 5239 5240 if (bp->port_stats.hw_stats) 5241 goto alloc_ext_stats; 5242 5243 bp->port_stats.len = BNXT_PORT_STATS_SIZE; 5244 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true); 5245 if (rc) 5246 return rc; 5247 5248 bp->flags |= BNXT_FLAG_PORT_STATS; 5249 5250alloc_ext_stats: 5251 /* Display extended statistics only if FW supports it */ 5252 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900) 5253 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) 5254 return 0; 5255 5256 if (bp->rx_port_stats_ext.hw_stats) 5257 goto alloc_tx_ext_stats; 5258 5259 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); 5260 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true); 5261 /* Extended stats are optional */ 5262 if (rc) 5263 return 0; 5264 5265alloc_tx_ext_stats: 5266 if (bp->tx_port_stats_ext.hw_stats) 5267 return 0; 5268 5269 if (bp->hwrm_spec_code >= 0x10902 || 5270 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) { 5271 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); 5272 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true); 5273 /* Extended stats are optional */ 5274 if (rc) 5275 return 0; 5276 } 5277 bp->flags |= BNXT_FLAG_PORT_STATS_EXT; 5278 return 0; 5279} 5280 5281static void bnxt_clear_ring_indices(struct bnxt *bp) 5282{ 5283 int i, j; 5284 5285 if (!bp->bnapi) 5286 return; 5287 5288 for (i = 0; i < bp->cp_nr_rings; i++) { 5289 struct bnxt_napi *bnapi = bp->bnapi[i]; 5290 struct bnxt_cp_ring_info *cpr; 5291 struct bnxt_rx_ring_info *rxr; 5292 struct bnxt_tx_ring_info *txr; 5293 5294 if (!bnapi) 5295 continue; 5296 5297 cpr = &bnapi->cp_ring; 5298 cpr->cp_raw_cons = 0; 5299 5300 bnxt_for_each_napi_tx(j, bnapi, txr) { 5301 txr->tx_prod = 0; 5302 txr->tx_cons = 0; 5303 txr->tx_hw_cons = 0; 5304 } 5305 5306 rxr = bnapi->rx_ring; 5307 if (rxr) { 5308 rxr->rx_prod = 0; 5309 rxr->rx_agg_prod = 0; 5310 rxr->rx_sw_agg_prod = 0; 5311 rxr->rx_next_cons = 0; 5312 } 5313 bnapi->events = 0; 5314 } 5315} 5316 5317void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 5318{ 5319 u8 type = fltr->type, flags = fltr->flags; 5320 5321 INIT_LIST_HEAD(&fltr->list); 5322 if ((type == BNXT_FLTR_TYPE_L2 && flags & BNXT_ACT_RING_DST) || 5323 (type == BNXT_FLTR_TYPE_NTUPLE && flags & BNXT_ACT_NO_AGING)) 5324 list_add_tail(&fltr->list, &bp->usr_fltr_list); 5325} 5326 5327void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 5328{ 5329 if (!list_empty(&fltr->list)) 5330 list_del_init(&fltr->list); 5331} 5332 5333static void bnxt_clear_usr_fltrs(struct bnxt *bp, bool all) 5334{ 5335 struct bnxt_filter_base *usr_fltr, *tmp; 5336 5337 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) { 5338 if (!all && usr_fltr->type == BNXT_FLTR_TYPE_L2) 5339 continue; 5340 bnxt_del_one_usr_fltr(bp, usr_fltr); 5341 } 5342} 5343 5344static void bnxt_del_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 5345{ 5346 hlist_del(&fltr->hash); 5347 bnxt_del_one_usr_fltr(bp, fltr); 5348 if (fltr->flags) { 5349 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap); 5350 bp->ntp_fltr_count--; 5351 } 5352 kfree(fltr); 5353} 5354 5355static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all) 5356{ 5357 int i; 5358 5359 netdev_assert_locked_or_invisible(bp->dev); 5360 5361 /* Under netdev instance lock and all our NAPIs have been disabled. 5362 * It's safe to delete the hash table. 5363 */ 5364 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 5365 struct hlist_head *head; 5366 struct hlist_node *tmp; 5367 struct bnxt_ntuple_filter *fltr; 5368 5369 head = &bp->ntp_fltr_hash_tbl[i]; 5370 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 5371 bnxt_del_l2_filter(bp, fltr->l2_fltr); 5372 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) || 5373 !list_empty(&fltr->base.list))) 5374 continue; 5375 bnxt_del_fltr(bp, &fltr->base); 5376 } 5377 } 5378 if (!all) 5379 return; 5380 5381 bitmap_free(bp->ntp_fltr_bmap); 5382 bp->ntp_fltr_bmap = NULL; 5383 bp->ntp_fltr_count = 0; 5384} 5385 5386static int bnxt_alloc_ntp_fltrs(struct bnxt *bp) 5387{ 5388 int i, rc = 0; 5389 5390 if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap) 5391 return 0; 5392 5393 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) 5394 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]); 5395 5396 bp->ntp_fltr_count = 0; 5397 bp->ntp_fltr_bmap = bitmap_zalloc(bp->max_fltr, GFP_KERNEL); 5398 5399 if (!bp->ntp_fltr_bmap) 5400 rc = -ENOMEM; 5401 5402 return rc; 5403} 5404 5405static void bnxt_free_l2_filters(struct bnxt *bp, bool all) 5406{ 5407 int i; 5408 5409 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) { 5410 struct hlist_head *head; 5411 struct hlist_node *tmp; 5412 struct bnxt_l2_filter *fltr; 5413 5414 head = &bp->l2_fltr_hash_tbl[i]; 5415 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 5416 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) || 5417 !list_empty(&fltr->base.list))) 5418 continue; 5419 bnxt_del_fltr(bp, &fltr->base); 5420 } 5421 } 5422} 5423 5424static void bnxt_init_l2_fltr_tbl(struct bnxt *bp) 5425{ 5426 int i; 5427 5428 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) 5429 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]); 5430 get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed)); 5431} 5432 5433static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init) 5434{ 5435 bnxt_free_vnic_attributes(bp); 5436 bnxt_free_tx_rings(bp); 5437 bnxt_free_rx_rings(bp); 5438 bnxt_free_cp_rings(bp); 5439 bnxt_free_all_cp_arrays(bp); 5440 bnxt_free_ntp_fltrs(bp, false); 5441 bnxt_free_l2_filters(bp, false); 5442 if (irq_re_init) { 5443 bnxt_free_ring_stats(bp); 5444 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) || 5445 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 5446 bnxt_free_port_stats(bp); 5447 bnxt_free_ring_grps(bp); 5448 bnxt_free_vnics(bp); 5449 kfree(bp->tx_ring_map); 5450 bp->tx_ring_map = NULL; 5451 kfree(bp->tx_ring); 5452 bp->tx_ring = NULL; 5453 kfree(bp->rx_ring); 5454 bp->rx_ring = NULL; 5455 kfree(bp->bnapi); 5456 bp->bnapi = NULL; 5457 } else { 5458 bnxt_clear_ring_indices(bp); 5459 } 5460} 5461 5462static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) 5463{ 5464 int i, j, rc, size, arr_size; 5465 void *bnapi; 5466 5467 if (irq_re_init) { 5468 /* Allocate bnapi mem pointer array and mem block for 5469 * all queues 5470 */ 5471 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) * 5472 bp->cp_nr_rings); 5473 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi)); 5474 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL); 5475 if (!bnapi) 5476 return -ENOMEM; 5477 5478 bp->bnapi = bnapi; 5479 bnapi += arr_size; 5480 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) { 5481 bp->bnapi[i] = bnapi; 5482 bp->bnapi[i]->index = i; 5483 bp->bnapi[i]->bp = bp; 5484 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 5485 struct bnxt_cp_ring_info *cpr = 5486 &bp->bnapi[i]->cp_ring; 5487 5488 cpr->cp_ring_struct.ring_mem.flags = 5489 BNXT_RMEM_RING_PTE_FLAG; 5490 } 5491 } 5492 5493 bp->rx_ring = kcalloc(bp->rx_nr_rings, 5494 sizeof(struct bnxt_rx_ring_info), 5495 GFP_KERNEL); 5496 if (!bp->rx_ring) 5497 return -ENOMEM; 5498 5499 for (i = 0; i < bp->rx_nr_rings; i++) { 5500 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 5501 5502 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 5503 rxr->rx_ring_struct.ring_mem.flags = 5504 BNXT_RMEM_RING_PTE_FLAG; 5505 rxr->rx_agg_ring_struct.ring_mem.flags = 5506 BNXT_RMEM_RING_PTE_FLAG; 5507 } else { 5508 rxr->rx_cpr = &bp->bnapi[i]->cp_ring; 5509 } 5510 rxr->bnapi = bp->bnapi[i]; 5511 bp->bnapi[i]->rx_ring = &bp->rx_ring[i]; 5512 } 5513 5514 bp->tx_ring = kcalloc(bp->tx_nr_rings, 5515 sizeof(struct bnxt_tx_ring_info), 5516 GFP_KERNEL); 5517 if (!bp->tx_ring) 5518 return -ENOMEM; 5519 5520 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16), 5521 GFP_KERNEL); 5522 5523 if (!bp->tx_ring_map) 5524 return -ENOMEM; 5525 5526 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 5527 j = 0; 5528 else 5529 j = bp->rx_nr_rings; 5530 5531 for (i = 0; i < bp->tx_nr_rings; i++) { 5532 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 5533 struct bnxt_napi *bnapi2; 5534 5535 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 5536 txr->tx_ring_struct.ring_mem.flags = 5537 BNXT_RMEM_RING_PTE_FLAG; 5538 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i; 5539 if (i >= bp->tx_nr_rings_xdp) { 5540 int k = j + BNXT_RING_TO_TC_OFF(bp, i); 5541 5542 bnapi2 = bp->bnapi[k]; 5543 txr->txq_index = i - bp->tx_nr_rings_xdp; 5544 txr->tx_napi_idx = 5545 BNXT_RING_TO_TC(bp, txr->txq_index); 5546 bnapi2->tx_ring[txr->tx_napi_idx] = txr; 5547 bnapi2->tx_int = bnxt_tx_int; 5548 } else { 5549 bnapi2 = bp->bnapi[j]; 5550 bnapi2->flags |= BNXT_NAPI_FLAG_XDP; 5551 bnapi2->tx_ring[0] = txr; 5552 bnapi2->tx_int = bnxt_tx_int_xdp; 5553 j++; 5554 } 5555 txr->bnapi = bnapi2; 5556 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 5557 txr->tx_cpr = &bnapi2->cp_ring; 5558 } 5559 5560 rc = bnxt_alloc_stats(bp); 5561 if (rc) 5562 goto alloc_mem_err; 5563 bnxt_init_stats(bp); 5564 5565 rc = bnxt_alloc_ntp_fltrs(bp); 5566 if (rc) 5567 goto alloc_mem_err; 5568 5569 rc = bnxt_alloc_vnics(bp); 5570 if (rc) 5571 goto alloc_mem_err; 5572 } 5573 5574 rc = bnxt_alloc_all_cp_arrays(bp); 5575 if (rc) 5576 goto alloc_mem_err; 5577 5578 bnxt_init_ring_struct(bp); 5579 5580 rc = bnxt_alloc_rx_rings(bp); 5581 if (rc) 5582 goto alloc_mem_err; 5583 5584 rc = bnxt_alloc_tx_rings(bp); 5585 if (rc) 5586 goto alloc_mem_err; 5587 5588 rc = bnxt_alloc_cp_rings(bp); 5589 if (rc) 5590 goto alloc_mem_err; 5591 5592 bp->vnic_info[BNXT_VNIC_DEFAULT].flags |= BNXT_VNIC_RSS_FLAG | 5593 BNXT_VNIC_MCAST_FLAG | 5594 BNXT_VNIC_UCAST_FLAG; 5595 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp) && (bp->flags & BNXT_FLAG_RFS)) 5596 bp->vnic_info[BNXT_VNIC_NTUPLE].flags |= 5597 BNXT_VNIC_RSS_FLAG | BNXT_VNIC_NTUPLE_FLAG; 5598 5599 rc = bnxt_alloc_vnic_attributes(bp); 5600 if (rc) 5601 goto alloc_mem_err; 5602 return 0; 5603 5604alloc_mem_err: 5605 bnxt_free_mem(bp, true); 5606 return rc; 5607} 5608 5609static void bnxt_disable_int(struct bnxt *bp) 5610{ 5611 int i; 5612 5613 if (!bp->bnapi) 5614 return; 5615 5616 for (i = 0; i < bp->cp_nr_rings; i++) { 5617 struct bnxt_napi *bnapi = bp->bnapi[i]; 5618 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5619 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 5620 5621 if (ring->fw_ring_id != INVALID_HW_RING_ID) 5622 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 5623 } 5624} 5625 5626static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n) 5627{ 5628 struct bnxt_napi *bnapi = bp->bnapi[n]; 5629 struct bnxt_cp_ring_info *cpr; 5630 5631 cpr = &bnapi->cp_ring; 5632 return cpr->cp_ring_struct.map_idx; 5633} 5634 5635static void bnxt_disable_int_sync(struct bnxt *bp) 5636{ 5637 int i; 5638 5639 if (!bp->irq_tbl) 5640 return; 5641 5642 atomic_inc(&bp->intr_sem); 5643 5644 bnxt_disable_int(bp); 5645 for (i = 0; i < bp->cp_nr_rings; i++) { 5646 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 5647 5648 synchronize_irq(bp->irq_tbl[map_idx].vector); 5649 } 5650} 5651 5652static void bnxt_enable_int(struct bnxt *bp) 5653{ 5654 int i; 5655 5656 atomic_set(&bp->intr_sem, 0); 5657 for (i = 0; i < bp->cp_nr_rings; i++) { 5658 struct bnxt_napi *bnapi = bp->bnapi[i]; 5659 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 5660 5661 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 5662 } 5663} 5664 5665int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size, 5666 bool async_only) 5667{ 5668 DECLARE_BITMAP(async_events_bmap, 256); 5669 u32 *events = (u32 *)async_events_bmap; 5670 struct hwrm_func_drv_rgtr_output *resp; 5671 struct hwrm_func_drv_rgtr_input *req; 5672 u32 flags; 5673 int rc, i; 5674 5675 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR); 5676 if (rc) 5677 return rc; 5678 5679 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE | 5680 FUNC_DRV_RGTR_REQ_ENABLES_VER | 5681 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 5682 5683 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX); 5684 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE; 5685 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) 5686 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT; 5687 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 5688 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT | 5689 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT; 5690 if (bp->fw_cap & BNXT_FW_CAP_NPAR_1_2) 5691 flags |= FUNC_DRV_RGTR_REQ_FLAGS_NPAR_1_2_SUPPORT; 5692 req->flags = cpu_to_le32(flags); 5693 req->ver_maj_8b = DRV_VER_MAJ; 5694 req->ver_min_8b = DRV_VER_MIN; 5695 req->ver_upd_8b = DRV_VER_UPD; 5696 req->ver_maj = cpu_to_le16(DRV_VER_MAJ); 5697 req->ver_min = cpu_to_le16(DRV_VER_MIN); 5698 req->ver_upd = cpu_to_le16(DRV_VER_UPD); 5699 5700 if (BNXT_PF(bp)) { 5701 u32 data[8]; 5702 int i; 5703 5704 memset(data, 0, sizeof(data)); 5705 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) { 5706 u16 cmd = bnxt_vf_req_snif[i]; 5707 unsigned int bit, idx; 5708 5709 if ((bp->fw_cap & BNXT_FW_CAP_LINK_ADMIN) && 5710 cmd == HWRM_PORT_PHY_QCFG) 5711 continue; 5712 5713 idx = cmd / 32; 5714 bit = cmd % 32; 5715 data[idx] |= 1 << bit; 5716 } 5717 5718 for (i = 0; i < 8; i++) 5719 req->vf_req_fwd[i] = cpu_to_le32(data[i]); 5720 5721 req->enables |= 5722 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD); 5723 } 5724 5725 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) 5726 req->flags |= cpu_to_le32( 5727 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE); 5728 5729 memset(async_events_bmap, 0, sizeof(async_events_bmap)); 5730 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { 5731 u16 event_id = bnxt_async_events_arr[i]; 5732 5733 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && 5734 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 5735 continue; 5736 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE && 5737 !bp->ptp_cfg) 5738 continue; 5739 __set_bit(bnxt_async_events_arr[i], async_events_bmap); 5740 } 5741 if (bmap && bmap_size) { 5742 for (i = 0; i < bmap_size; i++) { 5743 if (test_bit(i, bmap)) 5744 __set_bit(i, async_events_bmap); 5745 } 5746 } 5747 for (i = 0; i < 8; i++) 5748 req->async_event_fwd[i] |= cpu_to_le32(events[i]); 5749 5750 if (async_only) 5751 req->enables = 5752 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD); 5753 5754 resp = hwrm_req_hold(bp, req); 5755 rc = hwrm_req_send(bp, req); 5756 if (!rc) { 5757 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state); 5758 if (resp->flags & 5759 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED)) 5760 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; 5761 } 5762 hwrm_req_drop(bp, req); 5763 return rc; 5764} 5765 5766int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp) 5767{ 5768 struct hwrm_func_drv_unrgtr_input *req; 5769 int rc; 5770 5771 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state)) 5772 return 0; 5773 5774 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR); 5775 if (rc) 5776 return rc; 5777 return hwrm_req_send(bp, req); 5778} 5779 5780static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa); 5781 5782static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type) 5783{ 5784 struct hwrm_tunnel_dst_port_free_input *req; 5785 int rc; 5786 5787 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN && 5788 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID) 5789 return 0; 5790 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE && 5791 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID) 5792 return 0; 5793 5794 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE); 5795 if (rc) 5796 return rc; 5797 5798 req->tunnel_type = tunnel_type; 5799 5800 switch (tunnel_type) { 5801 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN: 5802 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id); 5803 bp->vxlan_port = 0; 5804 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 5805 break; 5806 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE: 5807 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id); 5808 bp->nge_port = 0; 5809 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 5810 break; 5811 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE: 5812 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id); 5813 bp->vxlan_gpe_port = 0; 5814 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID; 5815 break; 5816 default: 5817 break; 5818 } 5819 5820 rc = hwrm_req_send(bp, req); 5821 if (rc) 5822 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n", 5823 rc); 5824 if (bp->flags & BNXT_FLAG_TPA) 5825 bnxt_set_tpa(bp, true); 5826 return rc; 5827} 5828 5829static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port, 5830 u8 tunnel_type) 5831{ 5832 struct hwrm_tunnel_dst_port_alloc_output *resp; 5833 struct hwrm_tunnel_dst_port_alloc_input *req; 5834 int rc; 5835 5836 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC); 5837 if (rc) 5838 return rc; 5839 5840 req->tunnel_type = tunnel_type; 5841 req->tunnel_dst_port_val = port; 5842 5843 resp = hwrm_req_hold(bp, req); 5844 rc = hwrm_req_send(bp, req); 5845 if (rc) { 5846 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n", 5847 rc); 5848 goto err_out; 5849 } 5850 5851 switch (tunnel_type) { 5852 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN: 5853 bp->vxlan_port = port; 5854 bp->vxlan_fw_dst_port_id = 5855 le16_to_cpu(resp->tunnel_dst_port_id); 5856 break; 5857 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE: 5858 bp->nge_port = port; 5859 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id); 5860 break; 5861 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE: 5862 bp->vxlan_gpe_port = port; 5863 bp->vxlan_gpe_fw_dst_port_id = 5864 le16_to_cpu(resp->tunnel_dst_port_id); 5865 break; 5866 default: 5867 break; 5868 } 5869 if (bp->flags & BNXT_FLAG_TPA) 5870 bnxt_set_tpa(bp, true); 5871 5872err_out: 5873 hwrm_req_drop(bp, req); 5874 return rc; 5875} 5876 5877static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id) 5878{ 5879 struct hwrm_cfa_l2_set_rx_mask_input *req; 5880 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id]; 5881 int rc; 5882 5883 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK); 5884 if (rc) 5885 return rc; 5886 5887 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 5888 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) { 5889 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count); 5890 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping); 5891 } 5892 req->mask = cpu_to_le32(vnic->rx_mask); 5893 return hwrm_req_send_silent(bp, req); 5894} 5895 5896void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr) 5897{ 5898 if (!atomic_dec_and_test(&fltr->refcnt)) 5899 return; 5900 spin_lock_bh(&bp->ntp_fltr_lock); 5901 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) { 5902 spin_unlock_bh(&bp->ntp_fltr_lock); 5903 return; 5904 } 5905 hlist_del_rcu(&fltr->base.hash); 5906 bnxt_del_one_usr_fltr(bp, &fltr->base); 5907 if (fltr->base.flags) { 5908 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap); 5909 bp->ntp_fltr_count--; 5910 } 5911 spin_unlock_bh(&bp->ntp_fltr_lock); 5912 kfree_rcu(fltr, base.rcu); 5913} 5914 5915static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp, 5916 struct bnxt_l2_key *key, 5917 u32 idx) 5918{ 5919 struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx]; 5920 struct bnxt_l2_filter *fltr; 5921 5922 hlist_for_each_entry_rcu(fltr, head, base.hash) { 5923 struct bnxt_l2_key *l2_key = &fltr->l2_key; 5924 5925 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) && 5926 l2_key->vlan == key->vlan) 5927 return fltr; 5928 } 5929 return NULL; 5930} 5931 5932static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp, 5933 struct bnxt_l2_key *key, 5934 u32 idx) 5935{ 5936 struct bnxt_l2_filter *fltr = NULL; 5937 5938 rcu_read_lock(); 5939 fltr = __bnxt_lookup_l2_filter(bp, key, idx); 5940 if (fltr) 5941 atomic_inc(&fltr->refcnt); 5942 rcu_read_unlock(); 5943 return fltr; 5944} 5945 5946#define BNXT_IPV4_4TUPLE(bp, fkeys) \ 5947 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \ 5948 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) || \ 5949 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \ 5950 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4)) 5951 5952#define BNXT_IPV6_4TUPLE(bp, fkeys) \ 5953 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \ 5954 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) || \ 5955 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \ 5956 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6)) 5957 5958static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys) 5959{ 5960 if (fkeys->basic.n_proto == htons(ETH_P_IP)) { 5961 if (BNXT_IPV4_4TUPLE(bp, fkeys)) 5962 return sizeof(fkeys->addrs.v4addrs) + 5963 sizeof(fkeys->ports); 5964 5965 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4) 5966 return sizeof(fkeys->addrs.v4addrs); 5967 } 5968 5969 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) { 5970 if (BNXT_IPV6_4TUPLE(bp, fkeys)) 5971 return sizeof(fkeys->addrs.v6addrs) + 5972 sizeof(fkeys->ports); 5973 5974 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6) 5975 return sizeof(fkeys->addrs.v6addrs); 5976 } 5977 5978 return 0; 5979} 5980 5981static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys, 5982 const unsigned char *key) 5983{ 5984 u64 prefix = bp->toeplitz_prefix, hash = 0; 5985 struct bnxt_ipv4_tuple tuple4; 5986 struct bnxt_ipv6_tuple tuple6; 5987 int i, j, len = 0; 5988 u8 *four_tuple; 5989 5990 len = bnxt_get_rss_flow_tuple_len(bp, fkeys); 5991 if (!len) 5992 return 0; 5993 5994 if (fkeys->basic.n_proto == htons(ETH_P_IP)) { 5995 tuple4.v4addrs = fkeys->addrs.v4addrs; 5996 tuple4.ports = fkeys->ports; 5997 four_tuple = (unsigned char *)&tuple4; 5998 } else { 5999 tuple6.v6addrs = fkeys->addrs.v6addrs; 6000 tuple6.ports = fkeys->ports; 6001 four_tuple = (unsigned char *)&tuple6; 6002 } 6003 6004 for (i = 0, j = 8; i < len; i++, j++) { 6005 u8 byte = four_tuple[i]; 6006 int bit; 6007 6008 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) { 6009 if (byte & 0x80) 6010 hash ^= prefix; 6011 } 6012 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0; 6013 } 6014 6015 /* The valid part of the hash is in the upper 32 bits. */ 6016 return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK; 6017} 6018 6019#ifdef CONFIG_RFS_ACCEL 6020static struct bnxt_l2_filter * 6021bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key) 6022{ 6023 struct bnxt_l2_filter *fltr; 6024 u32 idx; 6025 6026 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 6027 BNXT_L2_FLTR_HASH_MASK; 6028 fltr = bnxt_lookup_l2_filter(bp, key, idx); 6029 return fltr; 6030} 6031#endif 6032 6033static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr, 6034 struct bnxt_l2_key *key, u32 idx) 6035{ 6036 struct hlist_head *head; 6037 6038 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr); 6039 fltr->l2_key.vlan = key->vlan; 6040 fltr->base.type = BNXT_FLTR_TYPE_L2; 6041 if (fltr->base.flags) { 6042 int bit_id; 6043 6044 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, 6045 bp->max_fltr, 0); 6046 if (bit_id < 0) 6047 return -ENOMEM; 6048 fltr->base.sw_id = (u16)bit_id; 6049 bp->ntp_fltr_count++; 6050 } 6051 head = &bp->l2_fltr_hash_tbl[idx]; 6052 hlist_add_head_rcu(&fltr->base.hash, head); 6053 bnxt_insert_usr_fltr(bp, &fltr->base); 6054 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state); 6055 atomic_set(&fltr->refcnt, 1); 6056 return 0; 6057} 6058 6059static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp, 6060 struct bnxt_l2_key *key, 6061 gfp_t gfp) 6062{ 6063 struct bnxt_l2_filter *fltr; 6064 u32 idx; 6065 int rc; 6066 6067 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 6068 BNXT_L2_FLTR_HASH_MASK; 6069 fltr = bnxt_lookup_l2_filter(bp, key, idx); 6070 if (fltr) 6071 return fltr; 6072 6073 fltr = kzalloc(sizeof(*fltr), gfp); 6074 if (!fltr) 6075 return ERR_PTR(-ENOMEM); 6076 spin_lock_bh(&bp->ntp_fltr_lock); 6077 rc = bnxt_init_l2_filter(bp, fltr, key, idx); 6078 spin_unlock_bh(&bp->ntp_fltr_lock); 6079 if (rc) { 6080 bnxt_del_l2_filter(bp, fltr); 6081 fltr = ERR_PTR(rc); 6082 } 6083 return fltr; 6084} 6085 6086struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp, 6087 struct bnxt_l2_key *key, 6088 u16 flags) 6089{ 6090 struct bnxt_l2_filter *fltr; 6091 u32 idx; 6092 int rc; 6093 6094 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) & 6095 BNXT_L2_FLTR_HASH_MASK; 6096 spin_lock_bh(&bp->ntp_fltr_lock); 6097 fltr = __bnxt_lookup_l2_filter(bp, key, idx); 6098 if (fltr) { 6099 fltr = ERR_PTR(-EEXIST); 6100 goto l2_filter_exit; 6101 } 6102 fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC); 6103 if (!fltr) { 6104 fltr = ERR_PTR(-ENOMEM); 6105 goto l2_filter_exit; 6106 } 6107 fltr->base.flags = flags; 6108 rc = bnxt_init_l2_filter(bp, fltr, key, idx); 6109 if (rc) { 6110 spin_unlock_bh(&bp->ntp_fltr_lock); 6111 bnxt_del_l2_filter(bp, fltr); 6112 return ERR_PTR(rc); 6113 } 6114 6115l2_filter_exit: 6116 spin_unlock_bh(&bp->ntp_fltr_lock); 6117 return fltr; 6118} 6119 6120static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx) 6121{ 6122#ifdef CONFIG_BNXT_SRIOV 6123 struct bnxt_vf_info *vf = &pf->vf[vf_idx]; 6124 6125 return vf->fw_fid; 6126#else 6127 return INVALID_HW_RING_ID; 6128#endif 6129} 6130 6131int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr) 6132{ 6133 struct hwrm_cfa_l2_filter_free_input *req; 6134 u16 target_id = 0xffff; 6135 int rc; 6136 6137 if (fltr->base.flags & BNXT_ACT_FUNC_DST) { 6138 struct bnxt_pf_info *pf = &bp->pf; 6139 6140 if (fltr->base.vf_idx >= pf->active_vfs) 6141 return -EINVAL; 6142 6143 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx); 6144 if (target_id == INVALID_HW_RING_ID) 6145 return -EINVAL; 6146 } 6147 6148 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE); 6149 if (rc) 6150 return rc; 6151 6152 req->target_id = cpu_to_le16(target_id); 6153 req->l2_filter_id = fltr->base.filter_id; 6154 return hwrm_req_send(bp, req); 6155} 6156 6157int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr) 6158{ 6159 struct hwrm_cfa_l2_filter_alloc_output *resp; 6160 struct hwrm_cfa_l2_filter_alloc_input *req; 6161 u16 target_id = 0xffff; 6162 int rc; 6163 6164 if (fltr->base.flags & BNXT_ACT_FUNC_DST) { 6165 struct bnxt_pf_info *pf = &bp->pf; 6166 6167 if (fltr->base.vf_idx >= pf->active_vfs) 6168 return -EINVAL; 6169 6170 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx); 6171 } 6172 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC); 6173 if (rc) 6174 return rc; 6175 6176 req->target_id = cpu_to_le16(target_id); 6177 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX); 6178 6179 if (!BNXT_CHIP_TYPE_NITRO_A0(bp)) 6180 req->flags |= 6181 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST); 6182 req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id); 6183 req->enables = 6184 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR | 6185 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID | 6186 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK); 6187 ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr); 6188 eth_broadcast_addr(req->l2_addr_mask); 6189 6190 if (fltr->l2_key.vlan) { 6191 req->enables |= 6192 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN | 6193 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK | 6194 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS); 6195 req->num_vlans = 1; 6196 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan); 6197 req->l2_ivlan_mask = cpu_to_le16(0xfff); 6198 } 6199 6200 resp = hwrm_req_hold(bp, req); 6201 rc = hwrm_req_send(bp, req); 6202 if (!rc) { 6203 fltr->base.filter_id = resp->l2_filter_id; 6204 set_bit(BNXT_FLTR_VALID, &fltr->base.state); 6205 } 6206 hwrm_req_drop(bp, req); 6207 return rc; 6208} 6209 6210int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp, 6211 struct bnxt_ntuple_filter *fltr) 6212{ 6213 struct hwrm_cfa_ntuple_filter_free_input *req; 6214 int rc; 6215 6216 set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state); 6217 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 6218 return 0; 6219 6220 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE); 6221 if (rc) 6222 return rc; 6223 6224 req->ntuple_filter_id = fltr->base.filter_id; 6225 return hwrm_req_send(bp, req); 6226} 6227 6228#define BNXT_NTP_FLTR_FLAGS \ 6229 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \ 6230 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \ 6231 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \ 6232 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \ 6233 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \ 6234 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \ 6235 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \ 6236 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \ 6237 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \ 6238 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \ 6239 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \ 6240 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \ 6241 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID) 6242 6243#define BNXT_NTP_TUNNEL_FLTR_FLAG \ 6244 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 6245 6246void bnxt_fill_ipv6_mask(__be32 mask[4]) 6247{ 6248 int i; 6249 6250 for (i = 0; i < 4; i++) 6251 mask[i] = cpu_to_be32(~0); 6252} 6253 6254static void 6255bnxt_cfg_rfs_ring_tbl_idx(struct bnxt *bp, 6256 struct hwrm_cfa_ntuple_filter_alloc_input *req, 6257 struct bnxt_ntuple_filter *fltr) 6258{ 6259 u16 rxq = fltr->base.rxq; 6260 6261 if (fltr->base.flags & BNXT_ACT_RSS_CTX) { 6262 struct ethtool_rxfh_context *ctx; 6263 struct bnxt_rss_ctx *rss_ctx; 6264 struct bnxt_vnic_info *vnic; 6265 6266 ctx = xa_load(&bp->dev->ethtool->rss_ctx, 6267 fltr->base.fw_vnic_id); 6268 if (ctx) { 6269 rss_ctx = ethtool_rxfh_context_priv(ctx); 6270 vnic = &rss_ctx->vnic; 6271 6272 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 6273 } 6274 return; 6275 } 6276 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) { 6277 struct bnxt_vnic_info *vnic; 6278 u32 enables; 6279 6280 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE]; 6281 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 6282 enables = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX; 6283 req->enables |= cpu_to_le32(enables); 6284 req->rfs_ring_tbl_idx = cpu_to_le16(rxq); 6285 } else { 6286 u32 flags; 6287 6288 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX; 6289 req->flags |= cpu_to_le32(flags); 6290 req->dst_id = cpu_to_le16(rxq); 6291 } 6292} 6293 6294int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp, 6295 struct bnxt_ntuple_filter *fltr) 6296{ 6297 struct hwrm_cfa_ntuple_filter_alloc_output *resp; 6298 struct hwrm_cfa_ntuple_filter_alloc_input *req; 6299 struct bnxt_flow_masks *masks = &fltr->fmasks; 6300 struct flow_keys *keys = &fltr->fkeys; 6301 struct bnxt_l2_filter *l2_fltr; 6302 struct bnxt_vnic_info *vnic; 6303 int rc; 6304 6305 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC); 6306 if (rc) 6307 return rc; 6308 6309 l2_fltr = fltr->l2_fltr; 6310 req->l2_filter_id = l2_fltr->base.filter_id; 6311 6312 if (fltr->base.flags & BNXT_ACT_DROP) { 6313 req->flags = 6314 cpu_to_le32(CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP); 6315 } else if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) { 6316 bnxt_cfg_rfs_ring_tbl_idx(bp, req, fltr); 6317 } else { 6318 vnic = &bp->vnic_info[fltr->base.rxq + 1]; 6319 req->dst_id = cpu_to_le16(vnic->fw_vnic_id); 6320 } 6321 req->enables |= cpu_to_le32(BNXT_NTP_FLTR_FLAGS); 6322 6323 req->ethertype = htons(ETH_P_IP); 6324 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4; 6325 req->ip_protocol = keys->basic.ip_proto; 6326 6327 if (keys->basic.n_proto == htons(ETH_P_IPV6)) { 6328 req->ethertype = htons(ETH_P_IPV6); 6329 req->ip_addr_type = 6330 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6; 6331 *(struct in6_addr *)&req->src_ipaddr[0] = keys->addrs.v6addrs.src; 6332 *(struct in6_addr *)&req->src_ipaddr_mask[0] = masks->addrs.v6addrs.src; 6333 *(struct in6_addr *)&req->dst_ipaddr[0] = keys->addrs.v6addrs.dst; 6334 *(struct in6_addr *)&req->dst_ipaddr_mask[0] = masks->addrs.v6addrs.dst; 6335 } else { 6336 req->src_ipaddr[0] = keys->addrs.v4addrs.src; 6337 req->src_ipaddr_mask[0] = masks->addrs.v4addrs.src; 6338 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst; 6339 req->dst_ipaddr_mask[0] = masks->addrs.v4addrs.dst; 6340 } 6341 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) { 6342 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG); 6343 req->tunnel_type = 6344 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL; 6345 } 6346 6347 req->src_port = keys->ports.src; 6348 req->src_port_mask = masks->ports.src; 6349 req->dst_port = keys->ports.dst; 6350 req->dst_port_mask = masks->ports.dst; 6351 6352 resp = hwrm_req_hold(bp, req); 6353 rc = hwrm_req_send(bp, req); 6354 if (!rc) 6355 fltr->base.filter_id = resp->ntuple_filter_id; 6356 hwrm_req_drop(bp, req); 6357 return rc; 6358} 6359 6360static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx, 6361 const u8 *mac_addr) 6362{ 6363 struct bnxt_l2_filter *fltr; 6364 struct bnxt_l2_key key; 6365 int rc; 6366 6367 ether_addr_copy(key.dst_mac_addr, mac_addr); 6368 key.vlan = 0; 6369 fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL); 6370 if (IS_ERR(fltr)) 6371 return PTR_ERR(fltr); 6372 6373 fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id; 6374 rc = bnxt_hwrm_l2_filter_alloc(bp, fltr); 6375 if (rc) 6376 bnxt_del_l2_filter(bp, fltr); 6377 else 6378 bp->vnic_info[vnic_id].l2_filters[idx] = fltr; 6379 return rc; 6380} 6381 6382static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp) 6383{ 6384 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */ 6385 6386 /* Any associated ntuple filters will also be cleared by firmware. */ 6387 for (i = 0; i < num_of_vnics; i++) { 6388 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 6389 6390 for (j = 0; j < vnic->uc_filter_count; j++) { 6391 struct bnxt_l2_filter *fltr = vnic->l2_filters[j]; 6392 6393 bnxt_hwrm_l2_filter_free(bp, fltr); 6394 bnxt_del_l2_filter(bp, fltr); 6395 } 6396 vnic->uc_filter_count = 0; 6397 } 6398} 6399 6400#define BNXT_DFLT_TUNL_TPA_BMAP \ 6401 (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \ 6402 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \ 6403 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6) 6404 6405static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp, 6406 struct hwrm_vnic_tpa_cfg_input *req) 6407{ 6408 u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP; 6409 6410 if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA)) 6411 return; 6412 6413 if (bp->vxlan_port) 6414 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN; 6415 if (bp->vxlan_gpe_port) 6416 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE; 6417 if (bp->nge_port) 6418 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE; 6419 6420 req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN); 6421 req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap); 6422} 6423 6424int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, struct bnxt_vnic_info *vnic, 6425 u32 tpa_flags) 6426{ 6427 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX; 6428 struct hwrm_vnic_tpa_cfg_input *req; 6429 int rc; 6430 6431 if (vnic->fw_vnic_id == INVALID_HW_RING_ID) 6432 return 0; 6433 6434 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG); 6435 if (rc) 6436 return rc; 6437 6438 if (tpa_flags) { 6439 u16 mss = bp->dev->mtu - 40; 6440 u32 nsegs, n, segs = 0, flags; 6441 6442 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA | 6443 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA | 6444 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE | 6445 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN | 6446 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ; 6447 if (tpa_flags & BNXT_FLAG_GRO) 6448 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO; 6449 6450 req->flags = cpu_to_le32(flags); 6451 6452 req->enables = 6453 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS | 6454 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS | 6455 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN); 6456 6457 /* Number of segs are log2 units, and first packet is not 6458 * included as part of this units. 6459 */ 6460 if (mss <= BNXT_RX_PAGE_SIZE) { 6461 n = BNXT_RX_PAGE_SIZE / mss; 6462 nsegs = (MAX_SKB_FRAGS - 1) * n; 6463 } else { 6464 n = mss / BNXT_RX_PAGE_SIZE; 6465 if (mss & (BNXT_RX_PAGE_SIZE - 1)) 6466 n++; 6467 nsegs = (MAX_SKB_FRAGS - n) / n; 6468 } 6469 6470 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6471 segs = MAX_TPA_SEGS_P5; 6472 max_aggs = bp->max_tpa; 6473 } else { 6474 segs = ilog2(nsegs); 6475 } 6476 req->max_agg_segs = cpu_to_le16(segs); 6477 req->max_aggs = cpu_to_le16(max_aggs); 6478 6479 req->min_agg_len = cpu_to_le32(512); 6480 bnxt_hwrm_vnic_update_tunl_tpa(bp, req); 6481 } 6482 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6483 6484 return hwrm_req_send(bp, req); 6485} 6486 6487static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring) 6488{ 6489 struct bnxt_ring_grp_info *grp_info; 6490 6491 grp_info = &bp->grp_info[ring->grp_idx]; 6492 return grp_info->cp_fw_ring_id; 6493} 6494 6495static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 6496{ 6497 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6498 return rxr->rx_cpr->cp_ring_struct.fw_ring_id; 6499 else 6500 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct); 6501} 6502 6503static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr) 6504{ 6505 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6506 return txr->tx_cpr->cp_ring_struct.fw_ring_id; 6507 else 6508 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct); 6509} 6510 6511static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp) 6512{ 6513 int entries; 6514 6515 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6516 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5; 6517 else 6518 entries = HW_HASH_INDEX_SIZE; 6519 6520 bp->rss_indir_tbl_entries = entries; 6521 bp->rss_indir_tbl = 6522 kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), GFP_KERNEL); 6523 if (!bp->rss_indir_tbl) 6524 return -ENOMEM; 6525 6526 return 0; 6527} 6528 6529void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp, 6530 struct ethtool_rxfh_context *rss_ctx) 6531{ 6532 u16 max_rings, max_entries, pad, i; 6533 u32 *rss_indir_tbl; 6534 6535 if (!bp->rx_nr_rings) 6536 return; 6537 6538 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6539 max_rings = bp->rx_nr_rings - 1; 6540 else 6541 max_rings = bp->rx_nr_rings; 6542 6543 max_entries = bnxt_get_rxfh_indir_size(bp->dev); 6544 if (rss_ctx) 6545 rss_indir_tbl = ethtool_rxfh_context_indir(rss_ctx); 6546 else 6547 rss_indir_tbl = &bp->rss_indir_tbl[0]; 6548 6549 for (i = 0; i < max_entries; i++) 6550 rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings); 6551 6552 pad = bp->rss_indir_tbl_entries - max_entries; 6553 if (pad) 6554 memset(&rss_indir_tbl[i], 0, pad * sizeof(*rss_indir_tbl)); 6555} 6556 6557static u16 bnxt_get_max_rss_ring(struct bnxt *bp) 6558{ 6559 u32 i, tbl_size, max_ring = 0; 6560 6561 if (!bp->rss_indir_tbl) 6562 return 0; 6563 6564 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 6565 for (i = 0; i < tbl_size; i++) 6566 max_ring = max(max_ring, bp->rss_indir_tbl[i]); 6567 return max_ring; 6568} 6569 6570int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) 6571{ 6572 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6573 if (!rx_rings) 6574 return 0; 6575 return bnxt_calc_nr_ring_pages(rx_rings - 1, 6576 BNXT_RSS_TABLE_ENTRIES_P5); 6577 } 6578 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 6579 return 2; 6580 return 1; 6581} 6582 6583static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) 6584{ 6585 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); 6586 u16 i, j; 6587 6588 /* Fill the RSS indirection table with ring group ids */ 6589 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { 6590 if (!no_rss) 6591 j = bp->rss_indir_tbl[i]; 6592 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); 6593 } 6594} 6595 6596static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, 6597 struct bnxt_vnic_info *vnic) 6598{ 6599 __le16 *ring_tbl = vnic->rss_table; 6600 struct bnxt_rx_ring_info *rxr; 6601 u16 tbl_size, i; 6602 6603 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 6604 6605 for (i = 0; i < tbl_size; i++) { 6606 u16 ring_id, j; 6607 6608 if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG) 6609 j = ethtool_rxfh_indir_default(i, bp->rx_nr_rings); 6610 else if (vnic->flags & BNXT_VNIC_RSSCTX_FLAG) 6611 j = ethtool_rxfh_context_indir(vnic->rss_ctx)[i]; 6612 else 6613 j = bp->rss_indir_tbl[i]; 6614 rxr = &bp->rx_ring[j]; 6615 6616 ring_id = rxr->rx_ring_struct.fw_ring_id; 6617 *ring_tbl++ = cpu_to_le16(ring_id); 6618 ring_id = bnxt_cp_ring_for_rx(bp, rxr); 6619 *ring_tbl++ = cpu_to_le16(ring_id); 6620 } 6621} 6622 6623static void 6624__bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, 6625 struct bnxt_vnic_info *vnic) 6626{ 6627 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6628 bnxt_fill_hw_rss_tbl_p5(bp, vnic); 6629 if (bp->flags & BNXT_FLAG_CHIP_P7) 6630 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; 6631 } else { 6632 bnxt_fill_hw_rss_tbl(bp, vnic); 6633 } 6634 6635 if (bp->rss_hash_delta) { 6636 req->hash_type = cpu_to_le32(bp->rss_hash_delta); 6637 if (bp->rss_hash_cfg & bp->rss_hash_delta) 6638 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE; 6639 else 6640 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE; 6641 } else { 6642 req->hash_type = cpu_to_le32(bp->rss_hash_cfg); 6643 } 6644 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT; 6645 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr); 6646 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr); 6647} 6648 6649static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct bnxt_vnic_info *vnic, 6650 bool set_rss) 6651{ 6652 struct hwrm_vnic_rss_cfg_input *req; 6653 int rc; 6654 6655 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) || 6656 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID) 6657 return 0; 6658 6659 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 6660 if (rc) 6661 return rc; 6662 6663 if (set_rss) 6664 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 6665 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6666 return hwrm_req_send(bp, req); 6667} 6668 6669static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, 6670 struct bnxt_vnic_info *vnic, bool set_rss) 6671{ 6672 struct hwrm_vnic_rss_cfg_input *req; 6673 dma_addr_t ring_tbl_map; 6674 u32 i, nr_ctxs; 6675 int rc; 6676 6677 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG); 6678 if (rc) 6679 return rc; 6680 6681 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6682 if (!set_rss) 6683 return hwrm_req_send(bp, req); 6684 6685 __bnxt_hwrm_vnic_set_rss(bp, req, vnic); 6686 ring_tbl_map = vnic->rss_table_dma_addr; 6687 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 6688 6689 hwrm_req_hold(bp, req); 6690 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) { 6691 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map); 6692 req->ring_table_pair_index = i; 6693 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]); 6694 rc = hwrm_req_send(bp, req); 6695 if (rc) 6696 goto exit; 6697 } 6698 6699exit: 6700 hwrm_req_drop(bp, req); 6701 return rc; 6702} 6703 6704static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp) 6705{ 6706 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 6707 struct hwrm_vnic_rss_qcfg_output *resp; 6708 struct hwrm_vnic_rss_qcfg_input *req; 6709 6710 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG)) 6711 return; 6712 6713 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6714 /* all contexts configured to same hash_type, zero always exists */ 6715 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6716 resp = hwrm_req_hold(bp, req); 6717 if (!hwrm_req_send(bp, req)) { 6718 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg; 6719 bp->rss_hash_delta = 0; 6720 } 6721 hwrm_req_drop(bp, req); 6722} 6723 6724static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, struct bnxt_vnic_info *vnic) 6725{ 6726 u16 hds_thresh = (u16)bp->dev->cfg_pending->hds_thresh; 6727 struct hwrm_vnic_plcmodes_cfg_input *req; 6728 int rc; 6729 6730 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG); 6731 if (rc) 6732 return rc; 6733 6734 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT); 6735 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID); 6736 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size); 6737 6738 if (!BNXT_RX_PAGE_MODE(bp) && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 6739 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 | 6740 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6); 6741 req->enables |= 6742 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID); 6743 req->hds_threshold = cpu_to_le16(hds_thresh); 6744 } 6745 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 6746 return hwrm_req_send(bp, req); 6747} 6748 6749static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, 6750 struct bnxt_vnic_info *vnic, 6751 u16 ctx_idx) 6752{ 6753 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req; 6754 6755 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE)) 6756 return; 6757 6758 req->rss_cos_lb_ctx_id = 6759 cpu_to_le16(vnic->fw_rss_cos_lb_ctx[ctx_idx]); 6760 6761 hwrm_req_send(bp, req); 6762 vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; 6763} 6764 6765static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp) 6766{ 6767 int i, j; 6768 6769 for (i = 0; i < bp->nr_vnics; i++) { 6770 struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; 6771 6772 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) { 6773 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID) 6774 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, j); 6775 } 6776 } 6777 bp->rsscos_nr_ctxs = 0; 6778} 6779 6780static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, 6781 struct bnxt_vnic_info *vnic, u16 ctx_idx) 6782{ 6783 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp; 6784 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req; 6785 int rc; 6786 6787 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); 6788 if (rc) 6789 return rc; 6790 6791 resp = hwrm_req_hold(bp, req); 6792 rc = hwrm_req_send(bp, req); 6793 if (!rc) 6794 vnic->fw_rss_cos_lb_ctx[ctx_idx] = 6795 le16_to_cpu(resp->rss_cos_lb_ctx_id); 6796 hwrm_req_drop(bp, req); 6797 6798 return rc; 6799} 6800 6801static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp) 6802{ 6803 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP) 6804 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE; 6805 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE; 6806} 6807 6808int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) 6809{ 6810 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 6811 struct hwrm_vnic_cfg_input *req; 6812 unsigned int ring = 0, grp_idx; 6813 u16 def_vlan = 0; 6814 int rc; 6815 6816 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG); 6817 if (rc) 6818 return rc; 6819 6820 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 6821 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; 6822 6823 req->default_rx_ring_id = 6824 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); 6825 req->default_cmpl_ring_id = 6826 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr)); 6827 req->enables = 6828 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID | 6829 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID); 6830 goto vnic_mru; 6831 } 6832 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP); 6833 /* Only RSS support for now TBD: COS & LB */ 6834 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) { 6835 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]); 6836 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 6837 VNIC_CFG_REQ_ENABLES_MRU); 6838 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) { 6839 req->rss_rule = cpu_to_le16(vnic0->fw_rss_cos_lb_ctx[0]); 6840 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE | 6841 VNIC_CFG_REQ_ENABLES_MRU); 6842 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE); 6843 } else { 6844 req->rss_rule = cpu_to_le16(0xffff); 6845 } 6846 6847 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && 6848 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) { 6849 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]); 6850 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE); 6851 } else { 6852 req->cos_rule = cpu_to_le16(0xffff); 6853 } 6854 6855 if (vnic->flags & BNXT_VNIC_RSS_FLAG) 6856 ring = 0; 6857 else if (vnic->flags & BNXT_VNIC_RFS_FLAG) 6858 ring = vnic->vnic_id - 1; 6859 else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) 6860 ring = bp->rx_nr_rings - 1; 6861 6862 grp_idx = bp->rx_ring[ring].bnapi->index; 6863 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); 6864 req->lb_rule = cpu_to_le16(0xffff); 6865vnic_mru: 6866 vnic->mru = bp->dev->mtu + VLAN_ETH_HLEN; 6867 req->mru = cpu_to_le16(vnic->mru); 6868 6869 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id); 6870#ifdef CONFIG_BNXT_SRIOV 6871 if (BNXT_VF(bp)) 6872 def_vlan = bp->vf.vlan; 6873#endif 6874 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan) 6875 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE); 6876 if (vnic->vnic_id == BNXT_VNIC_DEFAULT && bnxt_ulp_registered(bp->edev)) 6877 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp)); 6878 6879 return hwrm_req_send(bp, req); 6880} 6881 6882static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, 6883 struct bnxt_vnic_info *vnic) 6884{ 6885 if (vnic->fw_vnic_id != INVALID_HW_RING_ID) { 6886 struct hwrm_vnic_free_input *req; 6887 6888 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE)) 6889 return; 6890 6891 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 6892 6893 hwrm_req_send(bp, req); 6894 vnic->fw_vnic_id = INVALID_HW_RING_ID; 6895 } 6896} 6897 6898static void bnxt_hwrm_vnic_free(struct bnxt *bp) 6899{ 6900 u16 i; 6901 6902 for (i = 0; i < bp->nr_vnics; i++) 6903 bnxt_hwrm_vnic_free_one(bp, &bp->vnic_info[i]); 6904} 6905 6906int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic, 6907 unsigned int start_rx_ring_idx, 6908 unsigned int nr_rings) 6909{ 6910 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings; 6911 struct hwrm_vnic_alloc_output *resp; 6912 struct hwrm_vnic_alloc_input *req; 6913 int rc; 6914 6915 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC); 6916 if (rc) 6917 return rc; 6918 6919 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 6920 goto vnic_no_ring_grps; 6921 6922 /* map ring groups to this vnic */ 6923 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) { 6924 grp_idx = bp->rx_ring[i].bnapi->index; 6925 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) { 6926 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n", 6927 j, nr_rings); 6928 break; 6929 } 6930 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id; 6931 } 6932 6933vnic_no_ring_grps: 6934 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) 6935 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID; 6936 if (vnic->vnic_id == BNXT_VNIC_DEFAULT) 6937 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT); 6938 6939 resp = hwrm_req_hold(bp, req); 6940 rc = hwrm_req_send(bp, req); 6941 if (!rc) 6942 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id); 6943 hwrm_req_drop(bp, req); 6944 return rc; 6945} 6946 6947static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp) 6948{ 6949 struct hwrm_vnic_qcaps_output *resp; 6950 struct hwrm_vnic_qcaps_input *req; 6951 int rc; 6952 6953 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats); 6954 bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP; 6955 bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP; 6956 if (bp->hwrm_spec_code < 0x10600) 6957 return 0; 6958 6959 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS); 6960 if (rc) 6961 return rc; 6962 6963 resp = hwrm_req_hold(bp, req); 6964 rc = hwrm_req_send(bp, req); 6965 if (!rc) { 6966 u32 flags = le32_to_cpu(resp->flags); 6967 6968 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 6969 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP)) 6970 bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP; 6971 if (flags & 6972 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP) 6973 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP; 6974 6975 /* Older P5 fw before EXT_HW_STATS support did not set 6976 * VLAN_STRIP_CAP properly. 6977 */ 6978 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) || 6979 (BNXT_CHIP_P5(bp) && 6980 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))) 6981 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP; 6982 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP) 6983 bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA; 6984 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED) 6985 bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM; 6986 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported); 6987 if (bp->max_tpa_v2) { 6988 if (BNXT_CHIP_P5(bp)) 6989 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5; 6990 else 6991 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7; 6992 } 6993 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP) 6994 bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA; 6995 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP) 6996 bp->rss_cap |= BNXT_RSS_CAP_AH_V4_RSS_CAP; 6997 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP) 6998 bp->rss_cap |= BNXT_RSS_CAP_AH_V6_RSS_CAP; 6999 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP) 7000 bp->rss_cap |= BNXT_RSS_CAP_ESP_V4_RSS_CAP; 7001 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP) 7002 bp->rss_cap |= BNXT_RSS_CAP_ESP_V6_RSS_CAP; 7003 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPV6_FLOW_LABEL_CAP) 7004 bp->rss_cap |= BNXT_RSS_CAP_IPV6_FLOW_LABEL_RSS_CAP; 7005 if (flags & VNIC_QCAPS_RESP_FLAGS_RE_FLUSH_CAP) 7006 bp->fw_cap |= BNXT_FW_CAP_VNIC_RE_FLUSH; 7007 } 7008 hwrm_req_drop(bp, req); 7009 return rc; 7010} 7011 7012static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp) 7013{ 7014 struct hwrm_ring_grp_alloc_output *resp; 7015 struct hwrm_ring_grp_alloc_input *req; 7016 int rc; 7017 u16 i; 7018 7019 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7020 return 0; 7021 7022 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC); 7023 if (rc) 7024 return rc; 7025 7026 resp = hwrm_req_hold(bp, req); 7027 for (i = 0; i < bp->rx_nr_rings; i++) { 7028 unsigned int grp_idx = bp->rx_ring[i].bnapi->index; 7029 7030 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); 7031 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id); 7032 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id); 7033 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx); 7034 7035 rc = hwrm_req_send(bp, req); 7036 7037 if (rc) 7038 break; 7039 7040 bp->grp_info[grp_idx].fw_grp_id = 7041 le32_to_cpu(resp->ring_group_id); 7042 } 7043 hwrm_req_drop(bp, req); 7044 return rc; 7045} 7046 7047static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) 7048{ 7049 struct hwrm_ring_grp_free_input *req; 7050 u16 i; 7051 7052 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7053 return; 7054 7055 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE)) 7056 return; 7057 7058 hwrm_req_hold(bp, req); 7059 for (i = 0; i < bp->cp_nr_rings; i++) { 7060 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID) 7061 continue; 7062 req->ring_group_id = 7063 cpu_to_le32(bp->grp_info[i].fw_grp_id); 7064 7065 hwrm_req_send(bp, req); 7066 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID; 7067 } 7068 hwrm_req_drop(bp, req); 7069} 7070 7071static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, 7072 struct hwrm_ring_alloc_input *req, 7073 struct bnxt_ring_struct *ring) 7074{ 7075 struct bnxt_ring_grp_info *grp_info = &bp->grp_info[ring->grp_idx]; 7076 u32 enables = RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID | 7077 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID; 7078 7079 if (ring_type == HWRM_RING_ALLOC_AGG) { 7080 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; 7081 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); 7082 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); 7083 enables |= RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID; 7084 } else { 7085 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); 7086 if (NET_IP_ALIGN == 2) 7087 req->flags = 7088 cpu_to_le16(RING_ALLOC_REQ_FLAGS_RX_SOP_PAD); 7089 } 7090 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 7091 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 7092 req->enables |= cpu_to_le32(enables); 7093} 7094 7095static int hwrm_ring_alloc_send_msg(struct bnxt *bp, 7096 struct bnxt_ring_struct *ring, 7097 u32 ring_type, u32 map_index) 7098{ 7099 struct hwrm_ring_alloc_output *resp; 7100 struct hwrm_ring_alloc_input *req; 7101 struct bnxt_ring_mem_info *rmem = &ring->ring_mem; 7102 struct bnxt_ring_grp_info *grp_info; 7103 int rc, err = 0; 7104 u16 ring_id; 7105 7106 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC); 7107 if (rc) 7108 goto exit; 7109 7110 req->enables = 0; 7111 if (rmem->nr_pages > 1) { 7112 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map); 7113 /* Page size is in log2 units */ 7114 req->page_size = BNXT_PAGE_SHIFT; 7115 req->page_tbl_depth = 1; 7116 } else { 7117 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]); 7118 } 7119 req->fbo = 0; 7120 /* Association of ring index with doorbell index and MSIX number */ 7121 req->logical_id = cpu_to_le16(map_index); 7122 7123 switch (ring_type) { 7124 case HWRM_RING_ALLOC_TX: { 7125 struct bnxt_tx_ring_info *txr; 7126 u16 flags = 0; 7127 7128 txr = container_of(ring, struct bnxt_tx_ring_info, 7129 tx_ring_struct); 7130 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX; 7131 /* Association of transmit ring with completion ring */ 7132 grp_info = &bp->grp_info[ring->grp_idx]; 7133 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr)); 7134 req->length = cpu_to_le32(bp->tx_ring_mask + 1); 7135 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx); 7136 req->queue_id = cpu_to_le16(ring->queue_id); 7137 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL) 7138 req->cmpl_coal_cnt = 7139 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64; 7140 if ((bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) && bp->ptp_cfg) 7141 flags |= RING_ALLOC_REQ_FLAGS_TX_PKT_TS_CMPL_ENABLE; 7142 req->flags = cpu_to_le16(flags); 7143 break; 7144 } 7145 case HWRM_RING_ALLOC_RX: 7146 case HWRM_RING_ALLOC_AGG: 7147 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX; 7148 req->length = (ring_type == HWRM_RING_ALLOC_RX) ? 7149 cpu_to_le32(bp->rx_ring_mask + 1) : 7150 cpu_to_le32(bp->rx_agg_ring_mask + 1); 7151 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7152 bnxt_set_rx_ring_params_p5(bp, ring_type, req, ring); 7153 break; 7154 case HWRM_RING_ALLOC_CMPL: 7155 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; 7156 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 7157 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7158 /* Association of cp ring with nq */ 7159 grp_info = &bp->grp_info[map_index]; 7160 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id); 7161 req->cq_handle = cpu_to_le64(ring->handle); 7162 req->enables |= cpu_to_le32( 7163 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID); 7164 } else { 7165 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 7166 } 7167 break; 7168 case HWRM_RING_ALLOC_NQ: 7169 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ; 7170 req->length = cpu_to_le32(bp->cp_ring_mask + 1); 7171 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX; 7172 break; 7173 default: 7174 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n", 7175 ring_type); 7176 return -EINVAL; 7177 } 7178 7179 resp = hwrm_req_hold(bp, req); 7180 rc = hwrm_req_send(bp, req); 7181 err = le16_to_cpu(resp->error_code); 7182 ring_id = le16_to_cpu(resp->ring_id); 7183 hwrm_req_drop(bp, req); 7184 7185exit: 7186 if (rc || err) { 7187 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n", 7188 ring_type, rc, err); 7189 return -EIO; 7190 } 7191 ring->fw_ring_id = ring_id; 7192 return rc; 7193} 7194 7195static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx) 7196{ 7197 int rc; 7198 7199 if (BNXT_PF(bp)) { 7200 struct hwrm_func_cfg_input *req; 7201 7202 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 7203 if (rc) 7204 return rc; 7205 7206 req->fid = cpu_to_le16(0xffff); 7207 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 7208 req->async_event_cr = cpu_to_le16(idx); 7209 return hwrm_req_send(bp, req); 7210 } else { 7211 struct hwrm_func_vf_cfg_input *req; 7212 7213 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG); 7214 if (rc) 7215 return rc; 7216 7217 req->enables = 7218 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR); 7219 req->async_event_cr = cpu_to_le16(idx); 7220 return hwrm_req_send(bp, req); 7221 } 7222} 7223 7224static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db, 7225 u32 ring_type) 7226{ 7227 switch (ring_type) { 7228 case HWRM_RING_ALLOC_TX: 7229 db->db_ring_mask = bp->tx_ring_mask; 7230 break; 7231 case HWRM_RING_ALLOC_RX: 7232 db->db_ring_mask = bp->rx_ring_mask; 7233 break; 7234 case HWRM_RING_ALLOC_AGG: 7235 db->db_ring_mask = bp->rx_agg_ring_mask; 7236 break; 7237 case HWRM_RING_ALLOC_CMPL: 7238 case HWRM_RING_ALLOC_NQ: 7239 db->db_ring_mask = bp->cp_ring_mask; 7240 break; 7241 } 7242 if (bp->flags & BNXT_FLAG_CHIP_P7) { 7243 db->db_epoch_mask = db->db_ring_mask + 1; 7244 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); 7245 } 7246} 7247 7248static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type, 7249 u32 map_idx, u32 xid) 7250{ 7251 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7252 switch (ring_type) { 7253 case HWRM_RING_ALLOC_TX: 7254 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ; 7255 break; 7256 case HWRM_RING_ALLOC_RX: 7257 case HWRM_RING_ALLOC_AGG: 7258 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ; 7259 break; 7260 case HWRM_RING_ALLOC_CMPL: 7261 db->db_key64 = DBR_PATH_L2; 7262 break; 7263 case HWRM_RING_ALLOC_NQ: 7264 db->db_key64 = DBR_PATH_L2; 7265 break; 7266 } 7267 db->db_key64 |= (u64)xid << DBR_XID_SFT; 7268 7269 if (bp->flags & BNXT_FLAG_CHIP_P7) 7270 db->db_key64 |= DBR_VALID; 7271 7272 db->doorbell = bp->bar1 + bp->db_offset; 7273 } else { 7274 db->doorbell = bp->bar1 + map_idx * 0x80; 7275 switch (ring_type) { 7276 case HWRM_RING_ALLOC_TX: 7277 db->db_key32 = DB_KEY_TX; 7278 break; 7279 case HWRM_RING_ALLOC_RX: 7280 case HWRM_RING_ALLOC_AGG: 7281 db->db_key32 = DB_KEY_RX; 7282 break; 7283 case HWRM_RING_ALLOC_CMPL: 7284 db->db_key32 = DB_KEY_CP; 7285 break; 7286 } 7287 } 7288 bnxt_set_db_mask(bp, db, ring_type); 7289} 7290 7291static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp, 7292 struct bnxt_rx_ring_info *rxr) 7293{ 7294 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 7295 struct bnxt_napi *bnapi = rxr->bnapi; 7296 u32 type = HWRM_RING_ALLOC_RX; 7297 u32 map_idx = bnapi->index; 7298 int rc; 7299 7300 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7301 if (rc) 7302 return rc; 7303 7304 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id); 7305 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id; 7306 7307 return 0; 7308} 7309 7310static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp, 7311 struct bnxt_rx_ring_info *rxr) 7312{ 7313 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 7314 u32 type = HWRM_RING_ALLOC_AGG; 7315 u32 grp_idx = ring->grp_idx; 7316 u32 map_idx; 7317 int rc; 7318 7319 map_idx = grp_idx + bp->rx_nr_rings; 7320 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7321 if (rc) 7322 return rc; 7323 7324 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx, 7325 ring->fw_ring_id); 7326 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 7327 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 7328 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id; 7329 7330 return 0; 7331} 7332 7333static int bnxt_hwrm_cp_ring_alloc_p5(struct bnxt *bp, 7334 struct bnxt_cp_ring_info *cpr) 7335{ 7336 const u32 type = HWRM_RING_ALLOC_CMPL; 7337 struct bnxt_napi *bnapi = cpr->bnapi; 7338 struct bnxt_ring_struct *ring; 7339 u32 map_idx = bnapi->index; 7340 int rc; 7341 7342 ring = &cpr->cp_ring_struct; 7343 ring->handle = BNXT_SET_NQ_HDL(cpr); 7344 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7345 if (rc) 7346 return rc; 7347 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 7348 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons); 7349 return 0; 7350} 7351 7352static int bnxt_hwrm_tx_ring_alloc(struct bnxt *bp, 7353 struct bnxt_tx_ring_info *txr, u32 tx_idx) 7354{ 7355 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 7356 const u32 type = HWRM_RING_ALLOC_TX; 7357 int rc; 7358 7359 rc = hwrm_ring_alloc_send_msg(bp, ring, type, tx_idx); 7360 if (rc) 7361 return rc; 7362 bnxt_set_db(bp, &txr->tx_db, type, tx_idx, ring->fw_ring_id); 7363 return 0; 7364} 7365 7366static int bnxt_hwrm_ring_alloc(struct bnxt *bp) 7367{ 7368 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS); 7369 int i, rc = 0; 7370 u32 type; 7371 7372 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7373 type = HWRM_RING_ALLOC_NQ; 7374 else 7375 type = HWRM_RING_ALLOC_CMPL; 7376 for (i = 0; i < bp->cp_nr_rings; i++) { 7377 struct bnxt_napi *bnapi = bp->bnapi[i]; 7378 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7379 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 7380 u32 map_idx = ring->map_idx; 7381 unsigned int vector; 7382 7383 vector = bp->irq_tbl[map_idx].vector; 7384 disable_irq_nosync(vector); 7385 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); 7386 if (rc) { 7387 enable_irq(vector); 7388 goto err_out; 7389 } 7390 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); 7391 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 7392 enable_irq(vector); 7393 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id; 7394 7395 if (!i) { 7396 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id); 7397 if (rc) 7398 netdev_warn(bp->dev, "Failed to set async event completion ring.\n"); 7399 } 7400 } 7401 7402 for (i = 0; i < bp->tx_nr_rings; i++) { 7403 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; 7404 7405 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7406 rc = bnxt_hwrm_cp_ring_alloc_p5(bp, txr->tx_cpr); 7407 if (rc) 7408 goto err_out; 7409 } 7410 rc = bnxt_hwrm_tx_ring_alloc(bp, txr, i); 7411 if (rc) 7412 goto err_out; 7413 } 7414 7415 for (i = 0; i < bp->rx_nr_rings; i++) { 7416 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7417 7418 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); 7419 if (rc) 7420 goto err_out; 7421 /* If we have agg rings, post agg buffers first. */ 7422 if (!agg_rings) 7423 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 7424 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7425 rc = bnxt_hwrm_cp_ring_alloc_p5(bp, rxr->rx_cpr); 7426 if (rc) 7427 goto err_out; 7428 } 7429 } 7430 7431 if (agg_rings) { 7432 for (i = 0; i < bp->rx_nr_rings; i++) { 7433 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, &bp->rx_ring[i]); 7434 if (rc) 7435 goto err_out; 7436 } 7437 } 7438err_out: 7439 return rc; 7440} 7441 7442static void bnxt_cancel_dim(struct bnxt *bp) 7443{ 7444 int i; 7445 7446 /* DIM work is initialized in bnxt_enable_napi(). Proceed only 7447 * if NAPI is enabled. 7448 */ 7449 if (!bp->bnapi || test_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 7450 return; 7451 7452 /* Make sure NAPI sees that the VNIC is disabled */ 7453 synchronize_net(); 7454 for (i = 0; i < bp->rx_nr_rings; i++) { 7455 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 7456 struct bnxt_napi *bnapi = rxr->bnapi; 7457 7458 cancel_work_sync(&bnapi->cp_ring.dim.work); 7459 } 7460} 7461 7462static int hwrm_ring_free_send_msg(struct bnxt *bp, 7463 struct bnxt_ring_struct *ring, 7464 u32 ring_type, int cmpl_ring_id) 7465{ 7466 struct hwrm_ring_free_output *resp; 7467 struct hwrm_ring_free_input *req; 7468 u16 error_code = 0; 7469 int rc; 7470 7471 if (BNXT_NO_FW_ACCESS(bp)) 7472 return 0; 7473 7474 rc = hwrm_req_init(bp, req, HWRM_RING_FREE); 7475 if (rc) 7476 goto exit; 7477 7478 req->cmpl_ring = cpu_to_le16(cmpl_ring_id); 7479 req->ring_type = ring_type; 7480 req->ring_id = cpu_to_le16(ring->fw_ring_id); 7481 7482 resp = hwrm_req_hold(bp, req); 7483 rc = hwrm_req_send(bp, req); 7484 error_code = le16_to_cpu(resp->error_code); 7485 hwrm_req_drop(bp, req); 7486exit: 7487 if (rc || error_code) { 7488 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n", 7489 ring_type, rc, error_code); 7490 return -EIO; 7491 } 7492 return 0; 7493} 7494 7495static void bnxt_hwrm_tx_ring_free(struct bnxt *bp, 7496 struct bnxt_tx_ring_info *txr, 7497 bool close_path) 7498{ 7499 struct bnxt_ring_struct *ring = &txr->tx_ring_struct; 7500 u32 cmpl_ring_id; 7501 7502 if (ring->fw_ring_id == INVALID_HW_RING_ID) 7503 return; 7504 7505 cmpl_ring_id = close_path ? bnxt_cp_ring_for_tx(bp, txr) : 7506 INVALID_HW_RING_ID; 7507 hwrm_ring_free_send_msg(bp, ring, RING_FREE_REQ_RING_TYPE_TX, 7508 cmpl_ring_id); 7509 ring->fw_ring_id = INVALID_HW_RING_ID; 7510} 7511 7512static void bnxt_hwrm_rx_ring_free(struct bnxt *bp, 7513 struct bnxt_rx_ring_info *rxr, 7514 bool close_path) 7515{ 7516 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct; 7517 u32 grp_idx = rxr->bnapi->index; 7518 u32 cmpl_ring_id; 7519 7520 if (ring->fw_ring_id == INVALID_HW_RING_ID) 7521 return; 7522 7523 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7524 hwrm_ring_free_send_msg(bp, ring, 7525 RING_FREE_REQ_RING_TYPE_RX, 7526 close_path ? cmpl_ring_id : 7527 INVALID_HW_RING_ID); 7528 ring->fw_ring_id = INVALID_HW_RING_ID; 7529 bp->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID; 7530} 7531 7532static void bnxt_hwrm_rx_agg_ring_free(struct bnxt *bp, 7533 struct bnxt_rx_ring_info *rxr, 7534 bool close_path) 7535{ 7536 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct; 7537 u32 grp_idx = rxr->bnapi->index; 7538 u32 type, cmpl_ring_id; 7539 7540 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7541 type = RING_FREE_REQ_RING_TYPE_RX_AGG; 7542 else 7543 type = RING_FREE_REQ_RING_TYPE_RX; 7544 7545 if (ring->fw_ring_id == INVALID_HW_RING_ID) 7546 return; 7547 7548 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr); 7549 hwrm_ring_free_send_msg(bp, ring, type, 7550 close_path ? cmpl_ring_id : 7551 INVALID_HW_RING_ID); 7552 ring->fw_ring_id = INVALID_HW_RING_ID; 7553 bp->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID; 7554} 7555 7556static void bnxt_hwrm_cp_ring_free(struct bnxt *bp, 7557 struct bnxt_cp_ring_info *cpr) 7558{ 7559 struct bnxt_ring_struct *ring; 7560 7561 ring = &cpr->cp_ring_struct; 7562 if (ring->fw_ring_id == INVALID_HW_RING_ID) 7563 return; 7564 7565 hwrm_ring_free_send_msg(bp, ring, RING_FREE_REQ_RING_TYPE_L2_CMPL, 7566 INVALID_HW_RING_ID); 7567 ring->fw_ring_id = INVALID_HW_RING_ID; 7568} 7569 7570static void bnxt_clear_one_cp_ring(struct bnxt *bp, struct bnxt_cp_ring_info *cpr) 7571{ 7572 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; 7573 int i, size = ring->ring_mem.page_size; 7574 7575 cpr->cp_raw_cons = 0; 7576 cpr->toggle = 0; 7577 7578 for (i = 0; i < bp->cp_nr_pages; i++) 7579 if (cpr->cp_desc_ring[i]) 7580 memset(cpr->cp_desc_ring[i], 0, size); 7581} 7582 7583static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path) 7584{ 7585 u32 type; 7586 int i; 7587 7588 if (!bp->bnapi) 7589 return; 7590 7591 for (i = 0; i < bp->tx_nr_rings; i++) 7592 bnxt_hwrm_tx_ring_free(bp, &bp->tx_ring[i], close_path); 7593 7594 bnxt_cancel_dim(bp); 7595 for (i = 0; i < bp->rx_nr_rings; i++) { 7596 bnxt_hwrm_rx_ring_free(bp, &bp->rx_ring[i], close_path); 7597 bnxt_hwrm_rx_agg_ring_free(bp, &bp->rx_ring[i], close_path); 7598 } 7599 7600 /* The completion rings are about to be freed. After that the 7601 * IRQ doorbell will not work anymore. So we need to disable 7602 * IRQ here. 7603 */ 7604 bnxt_disable_int_sync(bp); 7605 7606 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7607 type = RING_FREE_REQ_RING_TYPE_NQ; 7608 else 7609 type = RING_FREE_REQ_RING_TYPE_L2_CMPL; 7610 for (i = 0; i < bp->cp_nr_rings; i++) { 7611 struct bnxt_napi *bnapi = bp->bnapi[i]; 7612 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 7613 struct bnxt_ring_struct *ring; 7614 int j; 7615 7616 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) 7617 bnxt_hwrm_cp_ring_free(bp, &cpr->cp_ring_arr[j]); 7618 7619 ring = &cpr->cp_ring_struct; 7620 if (ring->fw_ring_id != INVALID_HW_RING_ID) { 7621 hwrm_ring_free_send_msg(bp, ring, type, 7622 INVALID_HW_RING_ID); 7623 ring->fw_ring_id = INVALID_HW_RING_ID; 7624 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID; 7625 } 7626 } 7627} 7628 7629static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 7630 bool shared); 7631static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 7632 bool shared); 7633 7634static int bnxt_hwrm_get_rings(struct bnxt *bp) 7635{ 7636 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7637 struct hwrm_func_qcfg_output *resp; 7638 struct hwrm_func_qcfg_input *req; 7639 int rc; 7640 7641 if (bp->hwrm_spec_code < 0x10601) 7642 return 0; 7643 7644 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 7645 if (rc) 7646 return rc; 7647 7648 req->fid = cpu_to_le16(0xffff); 7649 resp = hwrm_req_hold(bp, req); 7650 rc = hwrm_req_send(bp, req); 7651 if (rc) { 7652 hwrm_req_drop(bp, req); 7653 return rc; 7654 } 7655 7656 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings); 7657 if (BNXT_NEW_RM(bp)) { 7658 u16 cp, stats; 7659 7660 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings); 7661 hw_resc->resv_hw_ring_grps = 7662 le32_to_cpu(resp->alloc_hw_ring_grps); 7663 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics); 7664 hw_resc->resv_rsscos_ctxs = le16_to_cpu(resp->alloc_rsscos_ctx); 7665 cp = le16_to_cpu(resp->alloc_cmpl_rings); 7666 stats = le16_to_cpu(resp->alloc_stat_ctx); 7667 hw_resc->resv_irqs = cp; 7668 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7669 int rx = hw_resc->resv_rx_rings; 7670 int tx = hw_resc->resv_tx_rings; 7671 7672 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7673 rx >>= 1; 7674 if (cp < (rx + tx)) { 7675 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false); 7676 if (rc) 7677 goto get_rings_exit; 7678 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7679 rx <<= 1; 7680 hw_resc->resv_rx_rings = rx; 7681 hw_resc->resv_tx_rings = tx; 7682 } 7683 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix); 7684 hw_resc->resv_hw_ring_grps = rx; 7685 } 7686 hw_resc->resv_cp_rings = cp; 7687 hw_resc->resv_stat_ctxs = stats; 7688 } 7689get_rings_exit: 7690 hwrm_req_drop(bp, req); 7691 return rc; 7692} 7693 7694int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings) 7695{ 7696 struct hwrm_func_qcfg_output *resp; 7697 struct hwrm_func_qcfg_input *req; 7698 int rc; 7699 7700 if (bp->hwrm_spec_code < 0x10601) 7701 return 0; 7702 7703 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 7704 if (rc) 7705 return rc; 7706 7707 req->fid = cpu_to_le16(fid); 7708 resp = hwrm_req_hold(bp, req); 7709 rc = hwrm_req_send(bp, req); 7710 if (!rc) 7711 *tx_rings = le16_to_cpu(resp->alloc_tx_rings); 7712 7713 hwrm_req_drop(bp, req); 7714 return rc; 7715} 7716 7717static bool bnxt_rfs_supported(struct bnxt *bp); 7718 7719static struct hwrm_func_cfg_input * 7720__bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7721{ 7722 struct hwrm_func_cfg_input *req; 7723 u32 enables = 0; 7724 7725 if (bnxt_hwrm_func_cfg_short_req_init(bp, &req)) 7726 return NULL; 7727 7728 req->fid = cpu_to_le16(0xffff); 7729 enables |= hwr->tx ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 7730 req->num_tx_rings = cpu_to_le16(hwr->tx); 7731 if (BNXT_NEW_RM(bp)) { 7732 enables |= hwr->rx ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0; 7733 enables |= hwr->stat ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 7734 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7735 enables |= hwr->cp ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0; 7736 enables |= hwr->cp_p5 ? 7737 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7738 } else { 7739 enables |= hwr->cp ? 7740 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7741 enables |= hwr->grp ? 7742 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 7743 } 7744 enables |= hwr->vnic ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0; 7745 enables |= hwr->rss_ctx ? FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 7746 0; 7747 req->num_rx_rings = cpu_to_le16(hwr->rx); 7748 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx); 7749 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7750 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5); 7751 req->num_msix = cpu_to_le16(hwr->cp); 7752 } else { 7753 req->num_cmpl_rings = cpu_to_le16(hwr->cp); 7754 req->num_hw_ring_grps = cpu_to_le16(hwr->grp); 7755 } 7756 req->num_stat_ctxs = cpu_to_le16(hwr->stat); 7757 req->num_vnics = cpu_to_le16(hwr->vnic); 7758 } 7759 req->enables = cpu_to_le32(enables); 7760 return req; 7761} 7762 7763static struct hwrm_func_vf_cfg_input * 7764__bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7765{ 7766 struct hwrm_func_vf_cfg_input *req; 7767 u32 enables = 0; 7768 7769 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG)) 7770 return NULL; 7771 7772 enables |= hwr->tx ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0; 7773 enables |= hwr->rx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS | 7774 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 7775 enables |= hwr->stat ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0; 7776 enables |= hwr->rss_ctx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0; 7777 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7778 enables |= hwr->cp_p5 ? 7779 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7780 } else { 7781 enables |= hwr->cp ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0; 7782 enables |= hwr->grp ? 7783 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0; 7784 } 7785 enables |= hwr->vnic ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0; 7786 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS; 7787 7788 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX); 7789 req->num_tx_rings = cpu_to_le16(hwr->tx); 7790 req->num_rx_rings = cpu_to_le16(hwr->rx); 7791 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx); 7792 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7793 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5); 7794 } else { 7795 req->num_cmpl_rings = cpu_to_le16(hwr->cp); 7796 req->num_hw_ring_grps = cpu_to_le16(hwr->grp); 7797 } 7798 req->num_stat_ctxs = cpu_to_le16(hwr->stat); 7799 req->num_vnics = cpu_to_le16(hwr->vnic); 7800 7801 req->enables = cpu_to_le32(enables); 7802 return req; 7803} 7804 7805static int 7806bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7807{ 7808 struct hwrm_func_cfg_input *req; 7809 int rc; 7810 7811 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr); 7812 if (!req) 7813 return -ENOMEM; 7814 7815 if (!req->enables) { 7816 hwrm_req_drop(bp, req); 7817 return 0; 7818 } 7819 7820 rc = hwrm_req_send(bp, req); 7821 if (rc) 7822 return rc; 7823 7824 if (bp->hwrm_spec_code < 0x10601) 7825 bp->hw_resc.resv_tx_rings = hwr->tx; 7826 7827 return bnxt_hwrm_get_rings(bp); 7828} 7829 7830static int 7831bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7832{ 7833 struct hwrm_func_vf_cfg_input *req; 7834 int rc; 7835 7836 if (!BNXT_NEW_RM(bp)) { 7837 bp->hw_resc.resv_tx_rings = hwr->tx; 7838 return 0; 7839 } 7840 7841 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr); 7842 if (!req) 7843 return -ENOMEM; 7844 7845 rc = hwrm_req_send(bp, req); 7846 if (rc) 7847 return rc; 7848 7849 return bnxt_hwrm_get_rings(bp); 7850} 7851 7852static int bnxt_hwrm_reserve_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7853{ 7854 if (BNXT_PF(bp)) 7855 return bnxt_hwrm_reserve_pf_rings(bp, hwr); 7856 else 7857 return bnxt_hwrm_reserve_vf_rings(bp, hwr); 7858} 7859 7860int bnxt_nq_rings_in_use(struct bnxt *bp) 7861{ 7862 return bp->cp_nr_rings + bnxt_get_ulp_msix_num(bp); 7863} 7864 7865static int bnxt_cp_rings_in_use(struct bnxt *bp) 7866{ 7867 int cp; 7868 7869 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7870 return bnxt_nq_rings_in_use(bp); 7871 7872 cp = bp->tx_nr_rings + bp->rx_nr_rings; 7873 return cp; 7874} 7875 7876static int bnxt_get_func_stat_ctxs(struct bnxt *bp) 7877{ 7878 return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp); 7879} 7880 7881static int bnxt_get_total_rss_ctxs(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7882{ 7883 if (!hwr->grp) 7884 return 0; 7885 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 7886 int rss_ctx = bnxt_get_nr_rss_ctxs(bp, hwr->grp); 7887 7888 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 7889 rss_ctx *= hwr->vnic; 7890 return rss_ctx; 7891 } 7892 if (BNXT_VF(bp)) 7893 return BNXT_VF_MAX_RSS_CTX; 7894 if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && bnxt_rfs_supported(bp)) 7895 return hwr->grp + 1; 7896 return 1; 7897} 7898 7899/* Check if a default RSS map needs to be setup. This function is only 7900 * used on older firmware that does not require reserving RX rings. 7901 */ 7902static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp) 7903{ 7904 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7905 7906 /* The RSS map is valid for RX rings set to resv_rx_rings */ 7907 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) { 7908 hw_resc->resv_rx_rings = bp->rx_nr_rings; 7909 if (!netif_is_rxfh_configured(bp->dev)) 7910 bnxt_set_dflt_rss_indir_tbl(bp, NULL); 7911 } 7912} 7913 7914static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings) 7915{ 7916 if (bp->flags & BNXT_FLAG_RFS) { 7917 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 7918 return 2 + bp->num_rss_ctx; 7919 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 7920 return rx_rings + 1; 7921 } 7922 return 1; 7923} 7924 7925static bool bnxt_need_reserve_rings(struct bnxt *bp) 7926{ 7927 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7928 int cp = bnxt_cp_rings_in_use(bp); 7929 int nq = bnxt_nq_rings_in_use(bp); 7930 int rx = bp->rx_nr_rings, stat; 7931 int vnic, grp = rx; 7932 7933 /* Old firmware does not need RX ring reservations but we still 7934 * need to setup a default RSS map when needed. With new firmware 7935 * we go through RX ring reservations first and then set up the 7936 * RSS map for the successfully reserved RX rings when needed. 7937 */ 7938 if (!BNXT_NEW_RM(bp)) 7939 bnxt_check_rss_tbl_no_rmgr(bp); 7940 7941 if (hw_resc->resv_tx_rings != bp->tx_nr_rings && 7942 bp->hwrm_spec_code >= 0x10601) 7943 return true; 7944 7945 if (!BNXT_NEW_RM(bp)) 7946 return false; 7947 7948 vnic = bnxt_get_total_vnics(bp, rx); 7949 7950 if (bp->flags & BNXT_FLAG_AGG_RINGS) 7951 rx <<= 1; 7952 stat = bnxt_get_func_stat_ctxs(bp); 7953 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp || 7954 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat || 7955 (hw_resc->resv_hw_ring_grps != grp && 7956 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))) 7957 return true; 7958 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) && 7959 hw_resc->resv_irqs != nq) 7960 return true; 7961 return false; 7962} 7963 7964static void bnxt_copy_reserved_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7965{ 7966 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 7967 7968 hwr->tx = hw_resc->resv_tx_rings; 7969 if (BNXT_NEW_RM(bp)) { 7970 hwr->rx = hw_resc->resv_rx_rings; 7971 hwr->cp = hw_resc->resv_irqs; 7972 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 7973 hwr->cp_p5 = hw_resc->resv_cp_rings; 7974 hwr->grp = hw_resc->resv_hw_ring_grps; 7975 hwr->vnic = hw_resc->resv_vnics; 7976 hwr->stat = hw_resc->resv_stat_ctxs; 7977 hwr->rss_ctx = hw_resc->resv_rsscos_ctxs; 7978 } 7979} 7980 7981static bool bnxt_rings_ok(struct bnxt *bp, struct bnxt_hw_rings *hwr) 7982{ 7983 return hwr->tx && hwr->rx && hwr->cp && hwr->grp && hwr->vnic && 7984 hwr->stat && (hwr->cp_p5 || !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)); 7985} 7986 7987static int bnxt_get_avail_msix(struct bnxt *bp, int num); 7988 7989static int __bnxt_reserve_rings(struct bnxt *bp) 7990{ 7991 struct bnxt_hw_rings hwr = {0}; 7992 int rx_rings, old_rx_rings, rc; 7993 int cp = bp->cp_nr_rings; 7994 int ulp_msix = 0; 7995 bool sh = false; 7996 int tx_cp; 7997 7998 if (!bnxt_need_reserve_rings(bp)) 7999 return 0; 8000 8001 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) { 8002 ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); 8003 if (!ulp_msix) 8004 bnxt_set_ulp_stat_ctxs(bp, 0); 8005 else 8006 bnxt_set_dflt_ulp_stat_ctxs(bp); 8007 8008 if (ulp_msix > bp->ulp_num_msix_want) 8009 ulp_msix = bp->ulp_num_msix_want; 8010 hwr.cp = cp + ulp_msix; 8011 } else { 8012 hwr.cp = bnxt_nq_rings_in_use(bp); 8013 } 8014 8015 hwr.tx = bp->tx_nr_rings; 8016 hwr.rx = bp->rx_nr_rings; 8017 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 8018 sh = true; 8019 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 8020 hwr.cp_p5 = hwr.rx + hwr.tx; 8021 8022 hwr.vnic = bnxt_get_total_vnics(bp, hwr.rx); 8023 8024 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8025 hwr.rx <<= 1; 8026 hwr.grp = bp->rx_nr_rings; 8027 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr); 8028 hwr.stat = bnxt_get_func_stat_ctxs(bp); 8029 old_rx_rings = bp->hw_resc.resv_rx_rings; 8030 8031 rc = bnxt_hwrm_reserve_rings(bp, &hwr); 8032 if (rc) 8033 return rc; 8034 8035 bnxt_copy_reserved_rings(bp, &hwr); 8036 8037 rx_rings = hwr.rx; 8038 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 8039 if (hwr.rx >= 2) { 8040 rx_rings = hwr.rx >> 1; 8041 } else { 8042 if (netif_running(bp->dev)) 8043 return -ENOMEM; 8044 8045 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 8046 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 8047 bp->dev->hw_features &= ~NETIF_F_LRO; 8048 bp->dev->features &= ~NETIF_F_LRO; 8049 bnxt_set_ring_params(bp); 8050 } 8051 } 8052 rx_rings = min_t(int, rx_rings, hwr.grp); 8053 hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings); 8054 if (bnxt_ulp_registered(bp->edev) && 8055 hwr.stat > bnxt_get_ulp_stat_ctxs(bp)) 8056 hwr.stat -= bnxt_get_ulp_stat_ctxs(bp); 8057 hwr.cp = min_t(int, hwr.cp, hwr.stat); 8058 rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh); 8059 if (bp->flags & BNXT_FLAG_AGG_RINGS) 8060 hwr.rx = rx_rings << 1; 8061 tx_cp = bnxt_num_tx_to_cp(bp, hwr.tx); 8062 hwr.cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings; 8063 if (hwr.tx != bp->tx_nr_rings) { 8064 netdev_warn(bp->dev, 8065 "Able to reserve only %d out of %d requested TX rings\n", 8066 hwr.tx, bp->tx_nr_rings); 8067 } 8068 bp->tx_nr_rings = hwr.tx; 8069 8070 /* If we cannot reserve all the RX rings, reset the RSS map only 8071 * if absolutely necessary 8072 */ 8073 if (rx_rings != bp->rx_nr_rings) { 8074 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n", 8075 rx_rings, bp->rx_nr_rings); 8076 if (netif_is_rxfh_configured(bp->dev) && 8077 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != 8078 bnxt_get_nr_rss_ctxs(bp, rx_rings) || 8079 bnxt_get_max_rss_ring(bp) >= rx_rings)) { 8080 netdev_warn(bp->dev, "RSS table entries reverting to default\n"); 8081 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 8082 } 8083 } 8084 bp->rx_nr_rings = rx_rings; 8085 bp->cp_nr_rings = hwr.cp; 8086 8087 if (!bnxt_rings_ok(bp, &hwr)) 8088 return -ENOMEM; 8089 8090 if (old_rx_rings != bp->hw_resc.resv_rx_rings && 8091 !netif_is_rxfh_configured(bp->dev)) 8092 bnxt_set_dflt_rss_indir_tbl(bp, NULL); 8093 8094 if (!bnxt_ulp_registered(bp->edev) && BNXT_NEW_RM(bp)) { 8095 int resv_msix, resv_ctx, ulp_ctxs; 8096 struct bnxt_hw_resc *hw_resc; 8097 8098 hw_resc = &bp->hw_resc; 8099 resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings; 8100 ulp_msix = min_t(int, resv_msix, ulp_msix); 8101 bnxt_set_ulp_msix_num(bp, ulp_msix); 8102 resv_ctx = hw_resc->resv_stat_ctxs - bp->cp_nr_rings; 8103 ulp_ctxs = min(resv_ctx, bnxt_get_ulp_stat_ctxs(bp)); 8104 bnxt_set_ulp_stat_ctxs(bp, ulp_ctxs); 8105 } 8106 8107 return rc; 8108} 8109 8110static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 8111{ 8112 struct hwrm_func_vf_cfg_input *req; 8113 u32 flags; 8114 8115 if (!BNXT_NEW_RM(bp)) 8116 return 0; 8117 8118 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr); 8119 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST | 8120 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST | 8121 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 8122 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 8123 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST | 8124 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST; 8125 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 8126 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 8127 8128 req->flags = cpu_to_le32(flags); 8129 return hwrm_req_send_silent(bp, req); 8130} 8131 8132static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 8133{ 8134 struct hwrm_func_cfg_input *req; 8135 u32 flags; 8136 8137 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr); 8138 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST; 8139 if (BNXT_NEW_RM(bp)) { 8140 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST | 8141 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST | 8142 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST | 8143 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST; 8144 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 8145 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST | 8146 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST; 8147 else 8148 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST; 8149 } 8150 8151 req->flags = cpu_to_le32(flags); 8152 return hwrm_req_send_silent(bp, req); 8153} 8154 8155static int bnxt_hwrm_check_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr) 8156{ 8157 if (bp->hwrm_spec_code < 0x10801) 8158 return 0; 8159 8160 if (BNXT_PF(bp)) 8161 return bnxt_hwrm_check_pf_rings(bp, hwr); 8162 8163 return bnxt_hwrm_check_vf_rings(bp, hwr); 8164} 8165 8166static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp) 8167{ 8168 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 8169 struct hwrm_ring_aggint_qcaps_output *resp; 8170 struct hwrm_ring_aggint_qcaps_input *req; 8171 int rc; 8172 8173 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS; 8174 coal_cap->num_cmpl_dma_aggr_max = 63; 8175 coal_cap->num_cmpl_dma_aggr_during_int_max = 63; 8176 coal_cap->cmpl_aggr_dma_tmr_max = 65535; 8177 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535; 8178 coal_cap->int_lat_tmr_min_max = 65535; 8179 coal_cap->int_lat_tmr_max_max = 65535; 8180 coal_cap->num_cmpl_aggr_int_max = 65535; 8181 coal_cap->timer_units = 80; 8182 8183 if (bp->hwrm_spec_code < 0x10902) 8184 return; 8185 8186 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS)) 8187 return; 8188 8189 resp = hwrm_req_hold(bp, req); 8190 rc = hwrm_req_send_silent(bp, req); 8191 if (!rc) { 8192 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params); 8193 coal_cap->nq_params = le32_to_cpu(resp->nq_params); 8194 coal_cap->num_cmpl_dma_aggr_max = 8195 le16_to_cpu(resp->num_cmpl_dma_aggr_max); 8196 coal_cap->num_cmpl_dma_aggr_during_int_max = 8197 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max); 8198 coal_cap->cmpl_aggr_dma_tmr_max = 8199 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max); 8200 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 8201 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max); 8202 coal_cap->int_lat_tmr_min_max = 8203 le16_to_cpu(resp->int_lat_tmr_min_max); 8204 coal_cap->int_lat_tmr_max_max = 8205 le16_to_cpu(resp->int_lat_tmr_max_max); 8206 coal_cap->num_cmpl_aggr_int_max = 8207 le16_to_cpu(resp->num_cmpl_aggr_int_max); 8208 coal_cap->timer_units = le16_to_cpu(resp->timer_units); 8209 } 8210 hwrm_req_drop(bp, req); 8211} 8212 8213static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec) 8214{ 8215 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 8216 8217 return usec * 1000 / coal_cap->timer_units; 8218} 8219 8220static void bnxt_hwrm_set_coal_params(struct bnxt *bp, 8221 struct bnxt_coal *hw_coal, 8222 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 8223{ 8224 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 8225 u16 val, tmr, max, flags = hw_coal->flags; 8226 u32 cmpl_params = coal_cap->cmpl_params; 8227 8228 max = hw_coal->bufs_per_record * 128; 8229 if (hw_coal->budget) 8230 max = hw_coal->bufs_per_record * hw_coal->budget; 8231 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max); 8232 8233 val = clamp_t(u16, hw_coal->coal_bufs, 1, max); 8234 req->num_cmpl_aggr_int = cpu_to_le16(val); 8235 8236 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max); 8237 req->num_cmpl_dma_aggr = cpu_to_le16(val); 8238 8239 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1, 8240 coal_cap->num_cmpl_dma_aggr_during_int_max); 8241 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val); 8242 8243 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks); 8244 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max); 8245 req->int_lat_tmr_max = cpu_to_le16(tmr); 8246 8247 /* min timer set to 1/2 of interrupt timer */ 8248 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) { 8249 val = tmr / 2; 8250 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max); 8251 req->int_lat_tmr_min = cpu_to_le16(val); 8252 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 8253 } 8254 8255 /* buf timer set to 1/4 of interrupt timer */ 8256 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max); 8257 req->cmpl_aggr_dma_tmr = cpu_to_le16(val); 8258 8259 if (cmpl_params & 8260 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) { 8261 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq); 8262 val = clamp_t(u16, tmr, 1, 8263 coal_cap->cmpl_aggr_dma_tmr_during_int_max); 8264 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val); 8265 req->enables |= 8266 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE); 8267 } 8268 8269 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) && 8270 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh) 8271 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE; 8272 req->flags = cpu_to_le16(flags); 8273 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES); 8274} 8275 8276static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi, 8277 struct bnxt_coal *hw_coal) 8278{ 8279 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req; 8280 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 8281 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 8282 u32 nq_params = coal_cap->nq_params; 8283 u16 tmr; 8284 int rc; 8285 8286 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN)) 8287 return 0; 8288 8289 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 8290 if (rc) 8291 return rc; 8292 8293 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id); 8294 req->flags = 8295 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ); 8296 8297 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2; 8298 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max); 8299 req->int_lat_tmr_min = cpu_to_le16(tmr); 8300 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE); 8301 return hwrm_req_send(bp, req); 8302} 8303 8304int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) 8305{ 8306 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx; 8307 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 8308 struct bnxt_coal coal; 8309 int rc; 8310 8311 /* Tick values in micro seconds. 8312 * 1 coal_buf x bufs_per_record = 1 completion record. 8313 */ 8314 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); 8315 8316 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; 8317 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; 8318 8319 if (!bnapi->rx_ring) 8320 return -ENODEV; 8321 8322 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 8323 if (rc) 8324 return rc; 8325 8326 bnxt_hwrm_set_coal_params(bp, &coal, req_rx); 8327 8328 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring)); 8329 8330 return hwrm_req_send(bp, req_rx); 8331} 8332 8333static int 8334bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi, 8335 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 8336{ 8337 u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring); 8338 8339 req->ring_id = cpu_to_le16(ring_id); 8340 return hwrm_req_send(bp, req); 8341} 8342 8343static int 8344bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi, 8345 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) 8346{ 8347 struct bnxt_tx_ring_info *txr; 8348 int i, rc; 8349 8350 bnxt_for_each_napi_tx(i, bnapi, txr) { 8351 u16 ring_id; 8352 8353 ring_id = bnxt_cp_ring_for_tx(bp, txr); 8354 req->ring_id = cpu_to_le16(ring_id); 8355 rc = hwrm_req_send(bp, req); 8356 if (rc) 8357 return rc; 8358 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 8359 return 0; 8360 } 8361 return 0; 8362} 8363 8364int bnxt_hwrm_set_coal(struct bnxt *bp) 8365{ 8366 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx; 8367 int i, rc; 8368 8369 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 8370 if (rc) 8371 return rc; 8372 8373 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); 8374 if (rc) { 8375 hwrm_req_drop(bp, req_rx); 8376 return rc; 8377 } 8378 8379 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx); 8380 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx); 8381 8382 hwrm_req_hold(bp, req_rx); 8383 hwrm_req_hold(bp, req_tx); 8384 for (i = 0; i < bp->cp_nr_rings; i++) { 8385 struct bnxt_napi *bnapi = bp->bnapi[i]; 8386 struct bnxt_coal *hw_coal; 8387 8388 if (!bnapi->rx_ring) 8389 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx); 8390 else 8391 rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx); 8392 if (rc) 8393 break; 8394 8395 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 8396 continue; 8397 8398 if (bnapi->rx_ring && bnapi->tx_ring[0]) { 8399 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx); 8400 if (rc) 8401 break; 8402 } 8403 if (bnapi->rx_ring) 8404 hw_coal = &bp->rx_coal; 8405 else 8406 hw_coal = &bp->tx_coal; 8407 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal); 8408 } 8409 hwrm_req_drop(bp, req_rx); 8410 hwrm_req_drop(bp, req_tx); 8411 return rc; 8412} 8413 8414static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp) 8415{ 8416 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL; 8417 struct hwrm_stat_ctx_free_input *req; 8418 int i; 8419 8420 if (!bp->bnapi) 8421 return; 8422 8423 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8424 return; 8425 8426 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE)) 8427 return; 8428 if (BNXT_FW_MAJ(bp) <= 20) { 8429 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) { 8430 hwrm_req_drop(bp, req); 8431 return; 8432 } 8433 hwrm_req_hold(bp, req0); 8434 } 8435 hwrm_req_hold(bp, req); 8436 for (i = 0; i < bp->cp_nr_rings; i++) { 8437 struct bnxt_napi *bnapi = bp->bnapi[i]; 8438 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 8439 8440 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) { 8441 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id); 8442 if (req0) { 8443 req0->stat_ctx_id = req->stat_ctx_id; 8444 hwrm_req_send(bp, req0); 8445 } 8446 hwrm_req_send(bp, req); 8447 8448 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; 8449 } 8450 } 8451 hwrm_req_drop(bp, req); 8452 if (req0) 8453 hwrm_req_drop(bp, req0); 8454} 8455 8456static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp) 8457{ 8458 struct hwrm_stat_ctx_alloc_output *resp; 8459 struct hwrm_stat_ctx_alloc_input *req; 8460 int rc, i; 8461 8462 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 8463 return 0; 8464 8465 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC); 8466 if (rc) 8467 return rc; 8468 8469 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size); 8470 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000); 8471 8472 resp = hwrm_req_hold(bp, req); 8473 for (i = 0; i < bp->cp_nr_rings; i++) { 8474 struct bnxt_napi *bnapi = bp->bnapi[i]; 8475 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 8476 8477 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map); 8478 8479 rc = hwrm_req_send(bp, req); 8480 if (rc) 8481 break; 8482 8483 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id); 8484 8485 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id; 8486 } 8487 hwrm_req_drop(bp, req); 8488 return rc; 8489} 8490 8491static int bnxt_hwrm_func_qcfg(struct bnxt *bp) 8492{ 8493 struct hwrm_func_qcfg_output *resp; 8494 struct hwrm_func_qcfg_input *req; 8495 u16 flags; 8496 int rc; 8497 8498 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG); 8499 if (rc) 8500 return rc; 8501 8502 req->fid = cpu_to_le16(0xffff); 8503 resp = hwrm_req_hold(bp, req); 8504 rc = hwrm_req_send(bp, req); 8505 if (rc) 8506 goto func_qcfg_exit; 8507 8508 flags = le16_to_cpu(resp->flags); 8509#ifdef CONFIG_BNXT_SRIOV 8510 if (BNXT_VF(bp)) { 8511 struct bnxt_vf_info *vf = &bp->vf; 8512 8513 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK; 8514 if (flags & FUNC_QCFG_RESP_FLAGS_TRUSTED_VF) 8515 vf->flags |= BNXT_VF_TRUST; 8516 else 8517 vf->flags &= ~BNXT_VF_TRUST; 8518 } else { 8519 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs); 8520 } 8521#endif 8522 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED | 8523 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) { 8524 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; 8525 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED) 8526 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; 8527 } 8528 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST)) 8529 bp->flags |= BNXT_FLAG_MULTI_HOST; 8530 8531 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED) 8532 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR; 8533 8534 if (flags & FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV) 8535 bp->fw_cap |= BNXT_FW_CAP_ENABLE_RDMA_SRIOV; 8536 if (resp->roce_bidi_opt_mode & 8537 FUNC_QCFG_RESP_ROCE_BIDI_OPT_MODE_DEDICATED) 8538 bp->cos0_cos1_shared = 1; 8539 else 8540 bp->cos0_cos1_shared = 0; 8541 8542 switch (resp->port_partition_type) { 8543 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0: 8544 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_2: 8545 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5: 8546 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0: 8547 bp->port_partition_type = resp->port_partition_type; 8548 break; 8549 } 8550 if (bp->hwrm_spec_code < 0x10707 || 8551 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB) 8552 bp->br_mode = BRIDGE_MODE_VEB; 8553 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA) 8554 bp->br_mode = BRIDGE_MODE_VEPA; 8555 else 8556 bp->br_mode = BRIDGE_MODE_UNDEF; 8557 8558 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured); 8559 if (!bp->max_mtu) 8560 bp->max_mtu = BNXT_MAX_MTU; 8561 8562 if (bp->db_size) 8563 goto func_qcfg_exit; 8564 8565 bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024; 8566 if (BNXT_CHIP_P5(bp)) { 8567 if (BNXT_PF(bp)) 8568 bp->db_offset = DB_PF_OFFSET_P5; 8569 else 8570 bp->db_offset = DB_VF_OFFSET_P5; 8571 } 8572 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 8573 1024); 8574 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) || 8575 bp->db_size <= bp->db_offset) 8576 bp->db_size = pci_resource_len(bp->pdev, 2); 8577 8578func_qcfg_exit: 8579 hwrm_req_drop(bp, req); 8580 return rc; 8581} 8582 8583static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm, 8584 u8 init_val, u8 init_offset, 8585 bool init_mask_set) 8586{ 8587 ctxm->init_value = init_val; 8588 ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET; 8589 if (init_mask_set) 8590 ctxm->init_offset = init_offset * 4; 8591 else 8592 ctxm->init_value = 0; 8593} 8594 8595static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max) 8596{ 8597 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8598 u16 type; 8599 8600 for (type = 0; type < ctx_max; type++) { 8601 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 8602 int n = 1; 8603 8604 if (!ctxm->max_entries || ctxm->pg_info) 8605 continue; 8606 8607 if (ctxm->instance_bmap) 8608 n = hweight32(ctxm->instance_bmap); 8609 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL); 8610 if (!ctxm->pg_info) 8611 return -ENOMEM; 8612 } 8613 return 0; 8614} 8615 8616static void bnxt_free_one_ctx_mem(struct bnxt *bp, 8617 struct bnxt_ctx_mem_type *ctxm, bool force); 8618 8619#define BNXT_CTX_INIT_VALID(flags) \ 8620 (!!((flags) & \ 8621 FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT)) 8622 8623static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp) 8624{ 8625 struct hwrm_func_backing_store_qcaps_v2_output *resp; 8626 struct hwrm_func_backing_store_qcaps_v2_input *req; 8627 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8628 u16 type, next_type = 0; 8629 int rc; 8630 8631 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); 8632 if (rc) 8633 return rc; 8634 8635 if (!ctx) { 8636 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 8637 if (!ctx) 8638 return -ENOMEM; 8639 bp->ctx = ctx; 8640 } 8641 8642 resp = hwrm_req_hold(bp, req); 8643 8644 for (type = 0; type < BNXT_CTX_V2_MAX; type = next_type) { 8645 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; 8646 u8 init_val, init_off, i; 8647 u32 max_entries; 8648 u16 entry_size; 8649 __le32 *p; 8650 u32 flags; 8651 8652 req->type = cpu_to_le16(type); 8653 rc = hwrm_req_send(bp, req); 8654 if (rc) 8655 goto ctx_done; 8656 flags = le32_to_cpu(resp->flags); 8657 next_type = le16_to_cpu(resp->next_valid_type); 8658 if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) { 8659 bnxt_free_one_ctx_mem(bp, ctxm, true); 8660 continue; 8661 } 8662 entry_size = le16_to_cpu(resp->entry_size); 8663 max_entries = le32_to_cpu(resp->max_num_entries); 8664 if (ctxm->mem_valid) { 8665 if (!(flags & BNXT_CTX_MEM_PERSIST) || 8666 ctxm->entry_size != entry_size || 8667 ctxm->max_entries != max_entries) 8668 bnxt_free_one_ctx_mem(bp, ctxm, true); 8669 else 8670 continue; 8671 } 8672 ctxm->type = type; 8673 ctxm->entry_size = entry_size; 8674 ctxm->flags = flags; 8675 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); 8676 ctxm->entry_multiple = resp->entry_multiple; 8677 ctxm->max_entries = max_entries; 8678 ctxm->min_entries = le32_to_cpu(resp->min_num_entries); 8679 init_val = resp->ctx_init_value; 8680 init_off = resp->ctx_init_offset; 8681 bnxt_init_ctx_initializer(ctxm, init_val, init_off, 8682 BNXT_CTX_INIT_VALID(flags)); 8683 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt, 8684 BNXT_MAX_SPLIT_ENTRY); 8685 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt; 8686 i++, p++) 8687 ctxm->split[i] = le32_to_cpu(*p); 8688 } 8689 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX); 8690 8691ctx_done: 8692 hwrm_req_drop(bp, req); 8693 return rc; 8694} 8695 8696static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp) 8697{ 8698 struct hwrm_func_backing_store_qcaps_output *resp; 8699 struct hwrm_func_backing_store_qcaps_input *req; 8700 int rc; 8701 8702 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || 8703 (bp->ctx && bp->ctx->flags & BNXT_CTX_FLAG_INITED)) 8704 return 0; 8705 8706 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) 8707 return bnxt_hwrm_func_backing_store_qcaps_v2(bp); 8708 8709 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS); 8710 if (rc) 8711 return rc; 8712 8713 resp = hwrm_req_hold(bp, req); 8714 rc = hwrm_req_send_silent(bp, req); 8715 if (!rc) { 8716 struct bnxt_ctx_mem_type *ctxm; 8717 struct bnxt_ctx_mem_info *ctx; 8718 u8 init_val, init_idx = 0; 8719 u16 init_mask; 8720 8721 ctx = bp->ctx; 8722 if (!ctx) { 8723 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 8724 if (!ctx) { 8725 rc = -ENOMEM; 8726 goto ctx_err; 8727 } 8728 bp->ctx = ctx; 8729 } 8730 init_val = resp->ctx_kind_initializer; 8731 init_mask = le16_to_cpu(resp->ctx_init_mask); 8732 8733 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8734 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries); 8735 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); 8736 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); 8737 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries); 8738 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size); 8739 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset, 8740 (init_mask & (1 << init_idx++)) != 0); 8741 8742 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8743 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); 8744 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries); 8745 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size); 8746 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset, 8747 (init_mask & (1 << init_idx++)) != 0); 8748 8749 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 8750 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); 8751 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries); 8752 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size); 8753 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset, 8754 (init_mask & (1 << init_idx++)) != 0); 8755 8756 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 8757 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries); 8758 ctxm->max_entries = ctxm->vnic_entries + 8759 le16_to_cpu(resp->vnic_max_ring_table_entries); 8760 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size); 8761 bnxt_init_ctx_initializer(ctxm, init_val, 8762 resp->vnic_init_offset, 8763 (init_mask & (1 << init_idx++)) != 0); 8764 8765 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 8766 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries); 8767 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size); 8768 bnxt_init_ctx_initializer(ctxm, init_val, 8769 resp->stat_init_offset, 8770 (init_mask & (1 << init_idx++)) != 0); 8771 8772 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 8773 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size); 8774 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring); 8775 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring); 8776 ctxm->entry_multiple = resp->tqm_entries_multiple; 8777 if (!ctxm->entry_multiple) 8778 ctxm->entry_multiple = 1; 8779 8780 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm)); 8781 8782 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 8783 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries); 8784 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size); 8785 ctxm->mrav_num_entries_units = 8786 le16_to_cpu(resp->mrav_num_entries_units); 8787 bnxt_init_ctx_initializer(ctxm, init_val, 8788 resp->mrav_init_offset, 8789 (init_mask & (1 << init_idx++)) != 0); 8790 8791 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 8792 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size); 8793 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries); 8794 8795 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; 8796 if (!ctx->tqm_fp_rings_count) 8797 ctx->tqm_fp_rings_count = bp->max_q; 8798 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) 8799 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; 8800 8801 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; 8802 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm)); 8803 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1; 8804 8805 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX); 8806 } else { 8807 rc = 0; 8808 } 8809ctx_err: 8810 hwrm_req_drop(bp, req); 8811 return rc; 8812} 8813 8814static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr, 8815 __le64 *pg_dir) 8816{ 8817 if (!rmem->nr_pages) 8818 return; 8819 8820 BNXT_SET_CTX_PAGE_ATTR(*pg_attr); 8821 if (rmem->depth >= 1) { 8822 if (rmem->depth == 2) 8823 *pg_attr |= 2; 8824 else 8825 *pg_attr |= 1; 8826 *pg_dir = cpu_to_le64(rmem->pg_tbl_map); 8827 } else { 8828 *pg_dir = cpu_to_le64(rmem->dma_arr[0]); 8829 } 8830} 8831 8832#define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \ 8833 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \ 8834 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \ 8835 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \ 8836 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \ 8837 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) 8838 8839static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables) 8840{ 8841 struct hwrm_func_backing_store_cfg_input *req; 8842 struct bnxt_ctx_mem_info *ctx = bp->ctx; 8843 struct bnxt_ctx_pg_info *ctx_pg; 8844 struct bnxt_ctx_mem_type *ctxm; 8845 void **__req = (void **)&req; 8846 u32 req_len = sizeof(*req); 8847 __le32 *num_entries; 8848 __le64 *pg_dir; 8849 u32 flags = 0; 8850 u8 *pg_attr; 8851 u32 ena; 8852 int rc; 8853 int i; 8854 8855 if (!ctx) 8856 return 0; 8857 8858 if (req_len > bp->hwrm_max_ext_req_len) 8859 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; 8860 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len); 8861 if (rc) 8862 return rc; 8863 8864 req->enables = cpu_to_le32(enables); 8865 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) { 8866 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 8867 ctx_pg = ctxm->pg_info; 8868 req->qp_num_entries = cpu_to_le32(ctx_pg->entries); 8869 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries); 8870 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries); 8871 req->qp_entry_size = cpu_to_le16(ctxm->entry_size); 8872 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8873 &req->qpc_pg_size_qpc_lvl, 8874 &req->qpc_page_dir); 8875 8876 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD) 8877 req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries); 8878 } 8879 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) { 8880 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 8881 ctx_pg = ctxm->pg_info; 8882 req->srq_num_entries = cpu_to_le32(ctx_pg->entries); 8883 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries); 8884 req->srq_entry_size = cpu_to_le16(ctxm->entry_size); 8885 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8886 &req->srq_pg_size_srq_lvl, 8887 &req->srq_page_dir); 8888 } 8889 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) { 8890 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 8891 ctx_pg = ctxm->pg_info; 8892 req->cq_num_entries = cpu_to_le32(ctx_pg->entries); 8893 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries); 8894 req->cq_entry_size = cpu_to_le16(ctxm->entry_size); 8895 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8896 &req->cq_pg_size_cq_lvl, 8897 &req->cq_page_dir); 8898 } 8899 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) { 8900 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 8901 ctx_pg = ctxm->pg_info; 8902 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries); 8903 req->vnic_num_ring_table_entries = 8904 cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries); 8905 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size); 8906 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8907 &req->vnic_pg_size_vnic_lvl, 8908 &req->vnic_page_dir); 8909 } 8910 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) { 8911 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 8912 ctx_pg = ctxm->pg_info; 8913 req->stat_num_entries = cpu_to_le32(ctxm->max_entries); 8914 req->stat_entry_size = cpu_to_le16(ctxm->entry_size); 8915 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8916 &req->stat_pg_size_stat_lvl, 8917 &req->stat_page_dir); 8918 } 8919 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) { 8920 u32 units; 8921 8922 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 8923 ctx_pg = ctxm->pg_info; 8924 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries); 8925 units = ctxm->mrav_num_entries_units; 8926 if (units) { 8927 u32 num_mr, num_ah = ctxm->mrav_av_entries; 8928 u32 entries; 8929 8930 num_mr = ctx_pg->entries - num_ah; 8931 entries = ((num_mr / units) << 16) | (num_ah / units); 8932 req->mrav_num_entries = cpu_to_le32(entries); 8933 flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT; 8934 } 8935 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size); 8936 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8937 &req->mrav_pg_size_mrav_lvl, 8938 &req->mrav_page_dir); 8939 } 8940 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) { 8941 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 8942 ctx_pg = ctxm->pg_info; 8943 req->tim_num_entries = cpu_to_le32(ctx_pg->entries); 8944 req->tim_entry_size = cpu_to_le16(ctxm->entry_size); 8945 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 8946 &req->tim_pg_size_tim_lvl, 8947 &req->tim_page_dir); 8948 } 8949 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 8950 for (i = 0, num_entries = &req->tqm_sp_num_entries, 8951 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl, 8952 pg_dir = &req->tqm_sp_page_dir, 8953 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP, 8954 ctx_pg = ctxm->pg_info; 8955 i < BNXT_MAX_TQM_RINGS; 8956 ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i], 8957 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { 8958 if (!(enables & ena)) 8959 continue; 8960 8961 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size); 8962 *num_entries = cpu_to_le32(ctx_pg->entries); 8963 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); 8964 } 8965 req->flags = cpu_to_le32(flags); 8966 return hwrm_req_send(bp, req); 8967} 8968 8969static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp, 8970 struct bnxt_ctx_pg_info *ctx_pg) 8971{ 8972 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 8973 8974 rmem->page_size = BNXT_PAGE_SIZE; 8975 rmem->pg_arr = ctx_pg->ctx_pg_arr; 8976 rmem->dma_arr = ctx_pg->ctx_dma_arr; 8977 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; 8978 if (rmem->depth >= 1) 8979 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; 8980 return bnxt_alloc_ring(bp, rmem); 8981} 8982 8983static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp, 8984 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, 8985 u8 depth, struct bnxt_ctx_mem_type *ctxm) 8986{ 8987 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 8988 int rc; 8989 8990 if (!mem_size) 8991 return -EINVAL; 8992 8993 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 8994 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { 8995 ctx_pg->nr_pages = 0; 8996 return -EINVAL; 8997 } 8998 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { 8999 int nr_tbls, i; 9000 9001 rmem->depth = 2; 9002 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg), 9003 GFP_KERNEL); 9004 if (!ctx_pg->ctx_pg_tbl) 9005 return -ENOMEM; 9006 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); 9007 rmem->nr_pages = nr_tbls; 9008 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 9009 if (rc) 9010 return rc; 9011 for (i = 0; i < nr_tbls; i++) { 9012 struct bnxt_ctx_pg_info *pg_tbl; 9013 9014 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); 9015 if (!pg_tbl) 9016 return -ENOMEM; 9017 ctx_pg->ctx_pg_tbl[i] = pg_tbl; 9018 rmem = &pg_tbl->ring_mem; 9019 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i]; 9020 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i]; 9021 rmem->depth = 1; 9022 rmem->nr_pages = MAX_CTX_PAGES; 9023 rmem->ctx_mem = ctxm; 9024 if (i == (nr_tbls - 1)) { 9025 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; 9026 9027 if (rem) 9028 rmem->nr_pages = rem; 9029 } 9030 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl); 9031 if (rc) 9032 break; 9033 } 9034 } else { 9035 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); 9036 if (rmem->nr_pages > 1 || depth) 9037 rmem->depth = 1; 9038 rmem->ctx_mem = ctxm; 9039 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg); 9040 } 9041 return rc; 9042} 9043 9044static size_t bnxt_copy_ctx_pg_tbls(struct bnxt *bp, 9045 struct bnxt_ctx_pg_info *ctx_pg, 9046 void *buf, size_t offset, size_t head, 9047 size_t tail) 9048{ 9049 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 9050 size_t nr_pages = ctx_pg->nr_pages; 9051 int page_size = rmem->page_size; 9052 size_t len = 0, total_len = 0; 9053 u16 depth = rmem->depth; 9054 9055 tail %= nr_pages * page_size; 9056 do { 9057 if (depth > 1) { 9058 int i = head / (page_size * MAX_CTX_PAGES); 9059 struct bnxt_ctx_pg_info *pg_tbl; 9060 9061 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 9062 rmem = &pg_tbl->ring_mem; 9063 } 9064 len = __bnxt_copy_ring(bp, rmem, buf, offset, head, tail); 9065 head += len; 9066 offset += len; 9067 total_len += len; 9068 if (head >= nr_pages * page_size) 9069 head = 0; 9070 } while (head != tail); 9071 return total_len; 9072} 9073 9074static void bnxt_free_ctx_pg_tbls(struct bnxt *bp, 9075 struct bnxt_ctx_pg_info *ctx_pg) 9076{ 9077 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; 9078 9079 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || 9080 ctx_pg->ctx_pg_tbl) { 9081 int i, nr_tbls = rmem->nr_pages; 9082 9083 for (i = 0; i < nr_tbls; i++) { 9084 struct bnxt_ctx_pg_info *pg_tbl; 9085 struct bnxt_ring_mem_info *rmem2; 9086 9087 pg_tbl = ctx_pg->ctx_pg_tbl[i]; 9088 if (!pg_tbl) 9089 continue; 9090 rmem2 = &pg_tbl->ring_mem; 9091 bnxt_free_ring(bp, rmem2); 9092 ctx_pg->ctx_pg_arr[i] = NULL; 9093 kfree(pg_tbl); 9094 ctx_pg->ctx_pg_tbl[i] = NULL; 9095 } 9096 kfree(ctx_pg->ctx_pg_tbl); 9097 ctx_pg->ctx_pg_tbl = NULL; 9098 } 9099 bnxt_free_ring(bp, rmem); 9100 ctx_pg->nr_pages = 0; 9101} 9102 9103static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp, 9104 struct bnxt_ctx_mem_type *ctxm, u32 entries, 9105 u8 pg_lvl) 9106{ 9107 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; 9108 int i, rc = 0, n = 1; 9109 u32 mem_size; 9110 9111 if (!ctxm->entry_size || !ctx_pg) 9112 return -EINVAL; 9113 if (ctxm->instance_bmap) 9114 n = hweight32(ctxm->instance_bmap); 9115 if (ctxm->entry_multiple) 9116 entries = roundup(entries, ctxm->entry_multiple); 9117 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries); 9118 mem_size = entries * ctxm->entry_size; 9119 for (i = 0; i < n && !rc; i++) { 9120 ctx_pg[i].entries = entries; 9121 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl, 9122 ctxm->init_value ? ctxm : NULL); 9123 } 9124 if (!rc) 9125 ctxm->mem_valid = 1; 9126 return rc; 9127} 9128 9129static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp, 9130 struct bnxt_ctx_mem_type *ctxm, 9131 bool last) 9132{ 9133 struct hwrm_func_backing_store_cfg_v2_input *req; 9134 u32 instance_bmap = ctxm->instance_bmap; 9135 int i, j, rc = 0, n = 1; 9136 __le32 *p; 9137 9138 if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info) 9139 return 0; 9140 9141 if (instance_bmap) 9142 n = hweight32(ctxm->instance_bmap); 9143 else 9144 instance_bmap = 1; 9145 9146 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2); 9147 if (rc) 9148 return rc; 9149 hwrm_req_hold(bp, req); 9150 req->type = cpu_to_le16(ctxm->type); 9151 req->entry_size = cpu_to_le16(ctxm->entry_size); 9152 if ((ctxm->flags & BNXT_CTX_MEM_PERSIST) && 9153 bnxt_bs_trace_avail(bp, ctxm->type)) { 9154 struct bnxt_bs_trace_info *bs_trace; 9155 u32 enables; 9156 9157 enables = FUNC_BACKING_STORE_CFG_V2_REQ_ENABLES_NEXT_BS_OFFSET; 9158 req->enables = cpu_to_le32(enables); 9159 bs_trace = &bp->bs_trace[bnxt_bstore_to_trace[ctxm->type]]; 9160 req->next_bs_offset = cpu_to_le32(bs_trace->last_offset); 9161 } 9162 req->subtype_valid_cnt = ctxm->split_entry_cnt; 9163 for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++) 9164 p[i] = cpu_to_le32(ctxm->split[i]); 9165 for (i = 0, j = 0; j < n && !rc; i++) { 9166 struct bnxt_ctx_pg_info *ctx_pg; 9167 9168 if (!(instance_bmap & (1 << i))) 9169 continue; 9170 req->instance = cpu_to_le16(i); 9171 ctx_pg = &ctxm->pg_info[j++]; 9172 if (!ctx_pg->entries) 9173 continue; 9174 req->num_entries = cpu_to_le32(ctx_pg->entries); 9175 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, 9176 &req->page_size_pbl_level, 9177 &req->page_dir); 9178 if (last && j == n) 9179 req->flags = 9180 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE); 9181 rc = hwrm_req_send(bp, req); 9182 } 9183 hwrm_req_drop(bp, req); 9184 return rc; 9185} 9186 9187static int bnxt_backing_store_cfg_v2(struct bnxt *bp) 9188{ 9189 struct bnxt_ctx_mem_info *ctx = bp->ctx; 9190 struct bnxt_ctx_mem_type *ctxm; 9191 u16 last_type = BNXT_CTX_INV; 9192 int rc = 0; 9193 u16 type; 9194 9195 for (type = BNXT_CTX_SRT; type <= BNXT_CTX_QPC; type++) { 9196 ctxm = &ctx->ctx_arr[type]; 9197 if (!bnxt_bs_trace_avail(bp, type)) 9198 continue; 9199 if (!ctxm->mem_valid) { 9200 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, 9201 ctxm->max_entries, 1); 9202 if (rc) { 9203 netdev_warn(bp->dev, "Unable to setup ctx page for type:0x%x.\n", 9204 type); 9205 continue; 9206 } 9207 bnxt_bs_trace_init(bp, ctxm); 9208 } 9209 last_type = type; 9210 } 9211 9212 if (last_type == BNXT_CTX_INV) { 9213 for (type = 0; type < BNXT_CTX_MAX; type++) { 9214 ctxm = &ctx->ctx_arr[type]; 9215 if (ctxm->mem_valid) 9216 last_type = type; 9217 } 9218 if (last_type == BNXT_CTX_INV) 9219 return 0; 9220 } 9221 ctx->ctx_arr[last_type].last = 1; 9222 9223 for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) { 9224 ctxm = &ctx->ctx_arr[type]; 9225 9226 if (!ctxm->mem_valid) 9227 continue; 9228 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last); 9229 if (rc) 9230 return rc; 9231 } 9232 return 0; 9233} 9234 9235/** 9236 * __bnxt_copy_ctx_mem - copy host context memory 9237 * @bp: The driver context 9238 * @ctxm: The pointer to the context memory type 9239 * @buf: The destination buffer or NULL to just obtain the length 9240 * @offset: The buffer offset to copy the data to 9241 * @head: The head offset of context memory to copy from 9242 * @tail: The tail offset (last byte + 1) of context memory to end the copy 9243 * 9244 * This function is called for debugging purposes to dump the host context 9245 * used by the chip. 9246 * 9247 * Return: Length of memory copied 9248 */ 9249static size_t __bnxt_copy_ctx_mem(struct bnxt *bp, 9250 struct bnxt_ctx_mem_type *ctxm, void *buf, 9251 size_t offset, size_t head, size_t tail) 9252{ 9253 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; 9254 size_t len = 0, total_len = 0; 9255 int i, n = 1; 9256 9257 if (!ctx_pg) 9258 return 0; 9259 9260 if (ctxm->instance_bmap) 9261 n = hweight32(ctxm->instance_bmap); 9262 for (i = 0; i < n; i++) { 9263 len = bnxt_copy_ctx_pg_tbls(bp, &ctx_pg[i], buf, offset, head, 9264 tail); 9265 offset += len; 9266 total_len += len; 9267 } 9268 return total_len; 9269} 9270 9271size_t bnxt_copy_ctx_mem(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm, 9272 void *buf, size_t offset) 9273{ 9274 size_t tail = ctxm->max_entries * ctxm->entry_size; 9275 9276 return __bnxt_copy_ctx_mem(bp, ctxm, buf, offset, 0, tail); 9277} 9278 9279static void bnxt_free_one_ctx_mem(struct bnxt *bp, 9280 struct bnxt_ctx_mem_type *ctxm, bool force) 9281{ 9282 struct bnxt_ctx_pg_info *ctx_pg; 9283 int i, n = 1; 9284 9285 ctxm->last = 0; 9286 9287 if (ctxm->mem_valid && !force && (ctxm->flags & BNXT_CTX_MEM_PERSIST)) 9288 return; 9289 9290 ctx_pg = ctxm->pg_info; 9291 if (ctx_pg) { 9292 if (ctxm->instance_bmap) 9293 n = hweight32(ctxm->instance_bmap); 9294 for (i = 0; i < n; i++) 9295 bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]); 9296 9297 kfree(ctx_pg); 9298 ctxm->pg_info = NULL; 9299 ctxm->mem_valid = 0; 9300 } 9301 memset(ctxm, 0, sizeof(*ctxm)); 9302} 9303 9304void bnxt_free_ctx_mem(struct bnxt *bp, bool force) 9305{ 9306 struct bnxt_ctx_mem_info *ctx = bp->ctx; 9307 u16 type; 9308 9309 if (!ctx) 9310 return; 9311 9312 for (type = 0; type < BNXT_CTX_V2_MAX; type++) 9313 bnxt_free_one_ctx_mem(bp, &ctx->ctx_arr[type], force); 9314 9315 ctx->flags &= ~BNXT_CTX_FLAG_INITED; 9316 if (force) { 9317 kfree(ctx); 9318 bp->ctx = NULL; 9319 } 9320} 9321 9322static int bnxt_alloc_ctx_mem(struct bnxt *bp) 9323{ 9324 struct bnxt_ctx_mem_type *ctxm; 9325 struct bnxt_ctx_mem_info *ctx; 9326 u32 l2_qps, qp1_qps, max_qps; 9327 u32 ena, entries_sp, entries; 9328 u32 srqs, max_srqs, min; 9329 u32 num_mr, num_ah; 9330 u32 extra_srqs = 0; 9331 u32 extra_qps = 0; 9332 u32 fast_qpmd_qps; 9333 u8 pg_lvl = 1; 9334 int i, rc; 9335 9336 rc = bnxt_hwrm_func_backing_store_qcaps(bp); 9337 if (rc) { 9338 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n", 9339 rc); 9340 return rc; 9341 } 9342 ctx = bp->ctx; 9343 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) 9344 return 0; 9345 9346 ena = 0; 9347 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 9348 goto skip_legacy; 9349 9350 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 9351 l2_qps = ctxm->qp_l2_entries; 9352 qp1_qps = ctxm->qp_qp1_entries; 9353 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries; 9354 max_qps = ctxm->max_entries; 9355 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 9356 srqs = ctxm->srq_l2_entries; 9357 max_srqs = ctxm->max_entries; 9358 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { 9359 pg_lvl = 2; 9360 if (BNXT_SW_RES_LMT(bp)) { 9361 extra_qps = max_qps - l2_qps - qp1_qps; 9362 extra_srqs = max_srqs - srqs; 9363 } else { 9364 extra_qps = min_t(u32, 65536, 9365 max_qps - l2_qps - qp1_qps); 9366 /* allocate extra qps if fw supports RoCE fast qp 9367 * destroy feature 9368 */ 9369 extra_qps += fast_qpmd_qps; 9370 extra_srqs = min_t(u32, 8192, max_srqs - srqs); 9371 } 9372 if (fast_qpmd_qps) 9373 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD; 9374 } 9375 9376 ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; 9377 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 9378 pg_lvl); 9379 if (rc) 9380 return rc; 9381 9382 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; 9383 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl); 9384 if (rc) 9385 return rc; 9386 9387 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; 9388 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries + 9389 extra_qps * 2, pg_lvl); 9390 if (rc) 9391 return rc; 9392 9393 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; 9394 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1); 9395 if (rc) 9396 return rc; 9397 9398 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; 9399 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1); 9400 if (rc) 9401 return rc; 9402 9403 if (!(bp->flags & BNXT_FLAG_ROCE_CAP)) 9404 goto skip_rdma; 9405 9406 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; 9407 if (BNXT_SW_RES_LMT(bp) && 9408 ctxm->split_entry_cnt == BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1) { 9409 num_ah = ctxm->mrav_av_entries; 9410 num_mr = ctxm->max_entries - num_ah; 9411 } else { 9412 /* 128K extra is needed to accommodate static AH context 9413 * allocation by f/w. 9414 */ 9415 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); 9416 num_ah = min_t(u32, num_mr, 1024 * 128); 9417 ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1; 9418 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah) 9419 ctxm->mrav_av_entries = num_ah; 9420 } 9421 9422 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2); 9423 if (rc) 9424 return rc; 9425 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; 9426 9427 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; 9428 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1); 9429 if (rc) 9430 return rc; 9431 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM; 9432 9433skip_rdma: 9434 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; 9435 min = ctxm->min_entries; 9436 entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps + 9437 2 * (extra_qps + qp1_qps) + min; 9438 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2); 9439 if (rc) 9440 return rc; 9441 9442 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; 9443 entries = l2_qps + 2 * (extra_qps + qp1_qps); 9444 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2); 9445 if (rc) 9446 return rc; 9447 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) 9448 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i; 9449 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES; 9450 9451skip_legacy: 9452 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) 9453 rc = bnxt_backing_store_cfg_v2(bp); 9454 else 9455 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena); 9456 if (rc) { 9457 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n", 9458 rc); 9459 return rc; 9460 } 9461 ctx->flags |= BNXT_CTX_FLAG_INITED; 9462 return 0; 9463} 9464 9465static int bnxt_hwrm_crash_dump_mem_cfg(struct bnxt *bp) 9466{ 9467 struct hwrm_dbg_crashdump_medium_cfg_input *req; 9468 u16 page_attr; 9469 int rc; 9470 9471 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR)) 9472 return 0; 9473 9474 rc = hwrm_req_init(bp, req, HWRM_DBG_CRASHDUMP_MEDIUM_CFG); 9475 if (rc) 9476 return rc; 9477 9478 if (BNXT_PAGE_SIZE == 0x2000) 9479 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_8K; 9480 else if (BNXT_PAGE_SIZE == 0x10000) 9481 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_64K; 9482 else 9483 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_4K; 9484 req->pg_size_lvl = cpu_to_le16(page_attr | 9485 bp->fw_crash_mem->ring_mem.depth); 9486 req->pbl = cpu_to_le64(bp->fw_crash_mem->ring_mem.pg_tbl_map); 9487 req->size = cpu_to_le32(bp->fw_crash_len); 9488 req->output_dest_flags = cpu_to_le16(BNXT_DBG_CR_DUMP_MDM_CFG_DDR); 9489 return hwrm_req_send(bp, req); 9490} 9491 9492static void bnxt_free_crash_dump_mem(struct bnxt *bp) 9493{ 9494 if (bp->fw_crash_mem) { 9495 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem); 9496 kfree(bp->fw_crash_mem); 9497 bp->fw_crash_mem = NULL; 9498 } 9499} 9500 9501static int bnxt_alloc_crash_dump_mem(struct bnxt *bp) 9502{ 9503 u32 mem_size = 0; 9504 int rc; 9505 9506 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR)) 9507 return 0; 9508 9509 rc = bnxt_hwrm_get_dump_len(bp, BNXT_DUMP_CRASH, &mem_size); 9510 if (rc) 9511 return rc; 9512 9513 mem_size = round_up(mem_size, 4); 9514 9515 /* keep and use the existing pages */ 9516 if (bp->fw_crash_mem && 9517 mem_size <= bp->fw_crash_mem->nr_pages * BNXT_PAGE_SIZE) 9518 goto alloc_done; 9519 9520 if (bp->fw_crash_mem) 9521 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem); 9522 else 9523 bp->fw_crash_mem = kzalloc(sizeof(*bp->fw_crash_mem), 9524 GFP_KERNEL); 9525 if (!bp->fw_crash_mem) 9526 return -ENOMEM; 9527 9528 rc = bnxt_alloc_ctx_pg_tbls(bp, bp->fw_crash_mem, mem_size, 1, NULL); 9529 if (rc) { 9530 bnxt_free_crash_dump_mem(bp); 9531 return rc; 9532 } 9533 9534alloc_done: 9535 bp->fw_crash_len = mem_size; 9536 return 0; 9537} 9538 9539int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all) 9540{ 9541 struct hwrm_func_resource_qcaps_output *resp; 9542 struct hwrm_func_resource_qcaps_input *req; 9543 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9544 int rc; 9545 9546 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS); 9547 if (rc) 9548 return rc; 9549 9550 req->fid = cpu_to_le16(0xffff); 9551 resp = hwrm_req_hold(bp, req); 9552 rc = hwrm_req_send_silent(bp, req); 9553 if (rc) 9554 goto hwrm_func_resc_qcaps_exit; 9555 9556 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs); 9557 if (!all) 9558 goto hwrm_func_resc_qcaps_exit; 9559 9560 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx); 9561 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 9562 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings); 9563 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 9564 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings); 9565 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 9566 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings); 9567 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 9568 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps); 9569 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps); 9570 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs); 9571 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 9572 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics); 9573 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 9574 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx); 9575 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 9576 9577 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 9578 u16 max_msix = le16_to_cpu(resp->max_msix); 9579 9580 hw_resc->max_nqs = max_msix; 9581 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; 9582 } 9583 9584 if (BNXT_PF(bp)) { 9585 struct bnxt_pf_info *pf = &bp->pf; 9586 9587 pf->vf_resv_strategy = 9588 le16_to_cpu(resp->vf_reservation_strategy); 9589 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC) 9590 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL; 9591 } 9592hwrm_func_resc_qcaps_exit: 9593 hwrm_req_drop(bp, req); 9594 return rc; 9595} 9596 9597static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp) 9598{ 9599 struct hwrm_port_mac_ptp_qcfg_output *resp; 9600 struct hwrm_port_mac_ptp_qcfg_input *req; 9601 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 9602 u8 flags; 9603 int rc; 9604 9605 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_PLUS(bp)) { 9606 rc = -ENODEV; 9607 goto no_ptp; 9608 } 9609 9610 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG); 9611 if (rc) 9612 goto no_ptp; 9613 9614 req->port_id = cpu_to_le16(bp->pf.port_id); 9615 resp = hwrm_req_hold(bp, req); 9616 rc = hwrm_req_send(bp, req); 9617 if (rc) 9618 goto exit; 9619 9620 flags = resp->flags; 9621 if (BNXT_CHIP_P5_AND_MINUS(bp) && 9622 !(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) { 9623 rc = -ENODEV; 9624 goto exit; 9625 } 9626 if (!ptp) { 9627 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL); 9628 if (!ptp) { 9629 rc = -ENOMEM; 9630 goto exit; 9631 } 9632 ptp->bp = bp; 9633 bp->ptp_cfg = ptp; 9634 } 9635 9636 if (flags & 9637 (PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK | 9638 PORT_MAC_PTP_QCFG_RESP_FLAGS_64B_PHC_TIME)) { 9639 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower); 9640 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper); 9641 } else if (BNXT_CHIP_P5(bp)) { 9642 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER; 9643 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER; 9644 } else { 9645 rc = -ENODEV; 9646 goto exit; 9647 } 9648 ptp->rtc_configured = 9649 (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0; 9650 rc = bnxt_ptp_init(bp); 9651 if (rc) 9652 netdev_warn(bp->dev, "PTP initialization failed.\n"); 9653exit: 9654 hwrm_req_drop(bp, req); 9655 if (!rc) 9656 return 0; 9657 9658no_ptp: 9659 bnxt_ptp_clear(bp); 9660 kfree(ptp); 9661 bp->ptp_cfg = NULL; 9662 return rc; 9663} 9664 9665static int __bnxt_hwrm_func_qcaps(struct bnxt *bp) 9666{ 9667 u32 flags, flags_ext, flags_ext2, flags_ext3; 9668 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 9669 struct hwrm_func_qcaps_output *resp; 9670 struct hwrm_func_qcaps_input *req; 9671 int rc; 9672 9673 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS); 9674 if (rc) 9675 return rc; 9676 9677 req->fid = cpu_to_le16(0xffff); 9678 resp = hwrm_req_hold(bp, req); 9679 rc = hwrm_req_send(bp, req); 9680 if (rc) 9681 goto hwrm_func_qcaps_exit; 9682 9683 flags = le32_to_cpu(resp->flags); 9684 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED) 9685 bp->flags |= BNXT_FLAG_ROCEV1_CAP; 9686 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) 9687 bp->flags |= BNXT_FLAG_ROCEV2_CAP; 9688 if (flags & FUNC_QCAPS_RESP_FLAGS_LINK_ADMIN_STATUS_SUPPORTED) 9689 bp->fw_cap |= BNXT_FW_CAP_LINK_ADMIN; 9690 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED) 9691 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; 9692 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE) 9693 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET; 9694 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) 9695 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; 9696 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE) 9697 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; 9698 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD) 9699 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; 9700 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) 9701 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; 9702 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED) 9703 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; 9704 9705 flags_ext = le32_to_cpu(resp->flags_ext); 9706 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) 9707 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; 9708 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED)) 9709 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS; 9710 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) 9711 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC; 9712 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) 9713 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; 9714 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) 9715 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH; 9716 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_NPAR_1_2_SUPPORTED) 9717 bp->fw_cap |= BNXT_FW_CAP_NPAR_1_2; 9718 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED)) 9719 bp->fw_cap |= BNXT_FW_CAP_DFLT_VLAN_TPID_PCP; 9720 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED) 9721 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2; 9722 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP) 9723 bp->flags |= BNXT_FLAG_TX_COAL_CMPL; 9724 9725 flags_ext2 = le32_to_cpu(resp->flags_ext2); 9726 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED) 9727 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS; 9728 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED) 9729 bp->flags |= BNXT_FLAG_UDP_GSO_CAP; 9730 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED) 9731 bp->fw_cap |= BNXT_FW_CAP_TX_TS_CMP; 9732 if (flags_ext2 & 9733 FUNC_QCAPS_RESP_FLAGS_EXT2_SW_MAX_RESOURCE_LIMITS_SUPPORTED) 9734 bp->fw_cap |= BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS; 9735 if (BNXT_PF(bp) && 9736 (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_ROCE_VF_RESOURCE_MGMT_SUPPORTED)) 9737 bp->fw_cap |= BNXT_FW_CAP_ROCE_VF_RESC_MGMT_SUPPORTED; 9738 9739 flags_ext3 = le32_to_cpu(resp->flags_ext3); 9740 if (flags_ext3 & FUNC_QCAPS_RESP_FLAGS_EXT3_ROCE_VF_DYN_ALLOC_SUPPORT) 9741 bp->fw_cap |= BNXT_FW_CAP_ROCE_VF_DYN_ALLOC_SUPPORT; 9742 if (flags_ext3 & FUNC_QCAPS_RESP_FLAGS_EXT3_MIRROR_ON_ROCE_SUPPORTED) 9743 bp->fw_cap |= BNXT_FW_CAP_MIRROR_ON_ROCE; 9744 9745 bp->tx_push_thresh = 0; 9746 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) && 9747 BNXT_FW_MAJ(bp) > 217) 9748 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH; 9749 9750 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx); 9751 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings); 9752 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings); 9753 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings); 9754 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps); 9755 if (!hw_resc->max_hw_ring_grps) 9756 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings; 9757 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs); 9758 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics); 9759 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx); 9760 9761 hw_resc->max_encap_records = le32_to_cpu(resp->max_encap_records); 9762 hw_resc->max_decap_records = le32_to_cpu(resp->max_decap_records); 9763 hw_resc->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows); 9764 hw_resc->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows); 9765 hw_resc->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows); 9766 hw_resc->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows); 9767 9768 if (BNXT_PF(bp)) { 9769 struct bnxt_pf_info *pf = &bp->pf; 9770 9771 pf->fw_fid = le16_to_cpu(resp->fid); 9772 pf->port_id = le16_to_cpu(resp->port_id); 9773 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN); 9774 pf->first_vf_id = le16_to_cpu(resp->first_vf_id); 9775 pf->max_vfs = le16_to_cpu(resp->max_vfs); 9776 bp->flags &= ~BNXT_FLAG_WOL_CAP; 9777 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED) 9778 bp->flags |= BNXT_FLAG_WOL_CAP; 9779 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) { 9780 bp->fw_cap |= BNXT_FW_CAP_PTP; 9781 } else { 9782 bnxt_ptp_clear(bp); 9783 kfree(bp->ptp_cfg); 9784 bp->ptp_cfg = NULL; 9785 } 9786 } else { 9787#ifdef CONFIG_BNXT_SRIOV 9788 struct bnxt_vf_info *vf = &bp->vf; 9789 9790 vf->fw_fid = le16_to_cpu(resp->fid); 9791 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN); 9792#endif 9793 } 9794 bp->tso_max_segs = le16_to_cpu(resp->max_tso_segs); 9795 9796hwrm_func_qcaps_exit: 9797 hwrm_req_drop(bp, req); 9798 return rc; 9799} 9800 9801static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp) 9802{ 9803 struct hwrm_dbg_qcaps_output *resp; 9804 struct hwrm_dbg_qcaps_input *req; 9805 int rc; 9806 9807 bp->fw_dbg_cap = 0; 9808 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS)) 9809 return; 9810 9811 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS); 9812 if (rc) 9813 return; 9814 9815 req->fid = cpu_to_le16(0xffff); 9816 resp = hwrm_req_hold(bp, req); 9817 rc = hwrm_req_send(bp, req); 9818 if (rc) 9819 goto hwrm_dbg_qcaps_exit; 9820 9821 bp->fw_dbg_cap = le32_to_cpu(resp->flags); 9822 9823hwrm_dbg_qcaps_exit: 9824 hwrm_req_drop(bp, req); 9825} 9826 9827static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp); 9828 9829int bnxt_hwrm_func_qcaps(struct bnxt *bp) 9830{ 9831 int rc; 9832 9833 rc = __bnxt_hwrm_func_qcaps(bp); 9834 if (rc) 9835 return rc; 9836 9837 bnxt_hwrm_dbg_qcaps(bp); 9838 9839 rc = bnxt_hwrm_queue_qportcfg(bp); 9840 if (rc) { 9841 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc); 9842 return rc; 9843 } 9844 if (bp->hwrm_spec_code >= 0x10803) { 9845 rc = bnxt_alloc_ctx_mem(bp); 9846 if (rc) 9847 return rc; 9848 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 9849 if (!rc) 9850 bp->fw_cap |= BNXT_FW_CAP_NEW_RM; 9851 } 9852 return 0; 9853} 9854 9855static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp) 9856{ 9857 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp; 9858 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req; 9859 u32 flags; 9860 int rc; 9861 9862 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW)) 9863 return 0; 9864 9865 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS); 9866 if (rc) 9867 return rc; 9868 9869 resp = hwrm_req_hold(bp, req); 9870 rc = hwrm_req_send(bp, req); 9871 if (rc) 9872 goto hwrm_cfa_adv_qcaps_exit; 9873 9874 flags = le32_to_cpu(resp->flags); 9875 if (flags & 9876 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED) 9877 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2; 9878 9879 if (flags & 9880 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED) 9881 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3; 9882 9883 if (flags & 9884 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED) 9885 bp->fw_cap |= BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO; 9886 9887hwrm_cfa_adv_qcaps_exit: 9888 hwrm_req_drop(bp, req); 9889 return rc; 9890} 9891 9892static int __bnxt_alloc_fw_health(struct bnxt *bp) 9893{ 9894 if (bp->fw_health) 9895 return 0; 9896 9897 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); 9898 if (!bp->fw_health) 9899 return -ENOMEM; 9900 9901 mutex_init(&bp->fw_health->lock); 9902 return 0; 9903} 9904 9905static int bnxt_alloc_fw_health(struct bnxt *bp) 9906{ 9907 int rc; 9908 9909 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && 9910 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 9911 return 0; 9912 9913 rc = __bnxt_alloc_fw_health(bp); 9914 if (rc) { 9915 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; 9916 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 9917 return rc; 9918 } 9919 9920 return 0; 9921} 9922 9923static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg) 9924{ 9925 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 + 9926 BNXT_GRCPF_REG_WINDOW_BASE_OUT + 9927 BNXT_FW_HEALTH_WIN_MAP_OFF); 9928} 9929 9930static void bnxt_inv_fw_health_reg(struct bnxt *bp) 9931{ 9932 struct bnxt_fw_health *fw_health = bp->fw_health; 9933 u32 reg_type; 9934 9935 if (!fw_health) 9936 return; 9937 9938 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); 9939 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 9940 fw_health->status_reliable = false; 9941 9942 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); 9943 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) 9944 fw_health->resets_reliable = false; 9945} 9946 9947static void bnxt_try_map_fw_health_reg(struct bnxt *bp) 9948{ 9949 void __iomem *hs; 9950 u32 status_loc; 9951 u32 reg_type; 9952 u32 sig; 9953 9954 if (bp->fw_health) 9955 bp->fw_health->status_reliable = false; 9956 9957 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC); 9958 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC); 9959 9960 sig = readl(hs + offsetof(struct hcomm_status, sig_ver)); 9961 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) { 9962 if (!bp->chip_num) { 9963 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE); 9964 bp->chip_num = readl(bp->bar0 + 9965 BNXT_FW_HEALTH_WIN_BASE + 9966 BNXT_GRC_REG_CHIP_NUM); 9967 } 9968 if (!BNXT_CHIP_P5_PLUS(bp)) 9969 return; 9970 9971 status_loc = BNXT_GRC_REG_STATUS_P5 | 9972 BNXT_FW_HEALTH_REG_TYPE_BAR0; 9973 } else { 9974 status_loc = readl(hs + offsetof(struct hcomm_status, 9975 fw_status_loc)); 9976 } 9977 9978 if (__bnxt_alloc_fw_health(bp)) { 9979 netdev_warn(bp->dev, "no memory for firmware status checks\n"); 9980 return; 9981 } 9982 9983 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc; 9984 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc); 9985 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) { 9986 __bnxt_map_fw_health_reg(bp, status_loc); 9987 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] = 9988 BNXT_FW_HEALTH_WIN_OFF(status_loc); 9989 } 9990 9991 bp->fw_health->status_reliable = true; 9992} 9993 9994static int bnxt_map_fw_health_regs(struct bnxt *bp) 9995{ 9996 struct bnxt_fw_health *fw_health = bp->fw_health; 9997 u32 reg_base = 0xffffffff; 9998 int i; 9999 10000 bp->fw_health->status_reliable = false; 10001 bp->fw_health->resets_reliable = false; 10002 /* Only pre-map the monitoring GRC registers using window 3 */ 10003 for (i = 0; i < 4; i++) { 10004 u32 reg = fw_health->regs[i]; 10005 10006 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) 10007 continue; 10008 if (reg_base == 0xffffffff) 10009 reg_base = reg & BNXT_GRC_BASE_MASK; 10010 if ((reg & BNXT_GRC_BASE_MASK) != reg_base) 10011 return -ERANGE; 10012 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); 10013 } 10014 bp->fw_health->status_reliable = true; 10015 bp->fw_health->resets_reliable = true; 10016 if (reg_base == 0xffffffff) 10017 return 0; 10018 10019 __bnxt_map_fw_health_reg(bp, reg_base); 10020 return 0; 10021} 10022 10023static void bnxt_remap_fw_health_regs(struct bnxt *bp) 10024{ 10025 if (!bp->fw_health) 10026 return; 10027 10028 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) { 10029 bp->fw_health->status_reliable = true; 10030 bp->fw_health->resets_reliable = true; 10031 } else { 10032 bnxt_try_map_fw_health_reg(bp); 10033 } 10034} 10035 10036static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp) 10037{ 10038 struct bnxt_fw_health *fw_health = bp->fw_health; 10039 struct hwrm_error_recovery_qcfg_output *resp; 10040 struct hwrm_error_recovery_qcfg_input *req; 10041 int rc, i; 10042 10043 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) 10044 return 0; 10045 10046 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG); 10047 if (rc) 10048 return rc; 10049 10050 resp = hwrm_req_hold(bp, req); 10051 rc = hwrm_req_send(bp, req); 10052 if (rc) 10053 goto err_recovery_out; 10054 fw_health->flags = le32_to_cpu(resp->flags); 10055 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) && 10056 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { 10057 rc = -EINVAL; 10058 goto err_recovery_out; 10059 } 10060 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq); 10061 fw_health->master_func_wait_dsecs = 10062 le32_to_cpu(resp->master_func_wait_period); 10063 fw_health->normal_func_wait_dsecs = 10064 le32_to_cpu(resp->normal_func_wait_period); 10065 fw_health->post_reset_wait_dsecs = 10066 le32_to_cpu(resp->master_func_wait_period_after_reset); 10067 fw_health->post_reset_max_wait_dsecs = 10068 le32_to_cpu(resp->max_bailout_time_after_reset); 10069 fw_health->regs[BNXT_FW_HEALTH_REG] = 10070 le32_to_cpu(resp->fw_health_status_reg); 10071 fw_health->regs[BNXT_FW_HEARTBEAT_REG] = 10072 le32_to_cpu(resp->fw_heartbeat_reg); 10073 fw_health->regs[BNXT_FW_RESET_CNT_REG] = 10074 le32_to_cpu(resp->fw_reset_cnt_reg); 10075 fw_health->regs[BNXT_FW_RESET_INPROG_REG] = 10076 le32_to_cpu(resp->reset_inprogress_reg); 10077 fw_health->fw_reset_inprog_reg_mask = 10078 le32_to_cpu(resp->reset_inprogress_reg_mask); 10079 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; 10080 if (fw_health->fw_reset_seq_cnt >= 16) { 10081 rc = -EINVAL; 10082 goto err_recovery_out; 10083 } 10084 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { 10085 fw_health->fw_reset_seq_regs[i] = 10086 le32_to_cpu(resp->reset_reg[i]); 10087 fw_health->fw_reset_seq_vals[i] = 10088 le32_to_cpu(resp->reset_reg_val[i]); 10089 fw_health->fw_reset_seq_delay_msec[i] = 10090 resp->delay_after_reset[i]; 10091 } 10092err_recovery_out: 10093 hwrm_req_drop(bp, req); 10094 if (!rc) 10095 rc = bnxt_map_fw_health_regs(bp); 10096 if (rc) 10097 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; 10098 return rc; 10099} 10100 10101static int bnxt_hwrm_func_reset(struct bnxt *bp) 10102{ 10103 struct hwrm_func_reset_input *req; 10104 int rc; 10105 10106 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET); 10107 if (rc) 10108 return rc; 10109 10110 req->enables = 0; 10111 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT); 10112 return hwrm_req_send(bp, req); 10113} 10114 10115static void bnxt_nvm_cfg_ver_get(struct bnxt *bp) 10116{ 10117 struct hwrm_nvm_get_dev_info_output nvm_info; 10118 10119 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info)) 10120 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d", 10121 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min, 10122 nvm_info.nvm_cfg_ver_upd); 10123} 10124 10125static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp) 10126{ 10127 struct hwrm_queue_qportcfg_output *resp; 10128 struct hwrm_queue_qportcfg_input *req; 10129 u8 i, j, *qptr; 10130 bool no_rdma; 10131 int rc = 0; 10132 10133 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG); 10134 if (rc) 10135 return rc; 10136 10137 resp = hwrm_req_hold(bp, req); 10138 rc = hwrm_req_send(bp, req); 10139 if (rc) 10140 goto qportcfg_exit; 10141 10142 if (!resp->max_configurable_queues) { 10143 rc = -EINVAL; 10144 goto qportcfg_exit; 10145 } 10146 bp->max_tc = resp->max_configurable_queues; 10147 bp->max_lltc = resp->max_configurable_lossless_queues; 10148 if (bp->max_tc > BNXT_MAX_QUEUE) 10149 bp->max_tc = BNXT_MAX_QUEUE; 10150 10151 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP); 10152 qptr = &resp->queue_id0; 10153 for (i = 0, j = 0; i < bp->max_tc; i++) { 10154 bp->q_info[j].queue_id = *qptr; 10155 bp->q_ids[i] = *qptr++; 10156 bp->q_info[j].queue_profile = *qptr++; 10157 bp->tc_to_qidx[j] = j; 10158 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) || 10159 (no_rdma && BNXT_PF(bp))) 10160 j++; 10161 } 10162 bp->max_q = bp->max_tc; 10163 bp->max_tc = max_t(u8, j, 1); 10164 10165 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG) 10166 bp->max_tc = 1; 10167 10168 if (bp->max_lltc > bp->max_tc) 10169 bp->max_lltc = bp->max_tc; 10170 10171qportcfg_exit: 10172 hwrm_req_drop(bp, req); 10173 return rc; 10174} 10175 10176static int bnxt_hwrm_poll(struct bnxt *bp) 10177{ 10178 struct hwrm_ver_get_input *req; 10179 int rc; 10180 10181 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 10182 if (rc) 10183 return rc; 10184 10185 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 10186 req->hwrm_intf_min = HWRM_VERSION_MINOR; 10187 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 10188 10189 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT); 10190 rc = hwrm_req_send(bp, req); 10191 return rc; 10192} 10193 10194static int bnxt_hwrm_ver_get(struct bnxt *bp) 10195{ 10196 struct hwrm_ver_get_output *resp; 10197 struct hwrm_ver_get_input *req; 10198 u16 fw_maj, fw_min, fw_bld, fw_rsv; 10199 u32 dev_caps_cfg, hwrm_ver; 10200 int rc, len, max_tmo_secs; 10201 10202 rc = hwrm_req_init(bp, req, HWRM_VER_GET); 10203 if (rc) 10204 return rc; 10205 10206 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 10207 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN; 10208 req->hwrm_intf_maj = HWRM_VERSION_MAJOR; 10209 req->hwrm_intf_min = HWRM_VERSION_MINOR; 10210 req->hwrm_intf_upd = HWRM_VERSION_UPDATE; 10211 10212 resp = hwrm_req_hold(bp, req); 10213 rc = hwrm_req_send(bp, req); 10214 if (rc) 10215 goto hwrm_ver_get_exit; 10216 10217 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output)); 10218 10219 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | 10220 resp->hwrm_intf_min_8b << 8 | 10221 resp->hwrm_intf_upd_8b; 10222 if (resp->hwrm_intf_maj_8b < 1) { 10223 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n", 10224 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 10225 resp->hwrm_intf_upd_8b); 10226 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n"); 10227 } 10228 10229 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 | 10230 HWRM_VERSION_UPDATE; 10231 10232 if (bp->hwrm_spec_code > hwrm_ver) 10233 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 10234 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR, 10235 HWRM_VERSION_UPDATE); 10236 else 10237 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d", 10238 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, 10239 resp->hwrm_intf_upd_8b); 10240 10241 fw_maj = le16_to_cpu(resp->hwrm_fw_major); 10242 if (bp->hwrm_spec_code > 0x10803 && fw_maj) { 10243 fw_min = le16_to_cpu(resp->hwrm_fw_minor); 10244 fw_bld = le16_to_cpu(resp->hwrm_fw_build); 10245 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch); 10246 len = FW_VER_STR_LEN; 10247 } else { 10248 fw_maj = resp->hwrm_fw_maj_8b; 10249 fw_min = resp->hwrm_fw_min_8b; 10250 fw_bld = resp->hwrm_fw_bld_8b; 10251 fw_rsv = resp->hwrm_fw_rsvd_8b; 10252 len = BC_HWRM_STR_LEN; 10253 } 10254 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); 10255 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, 10256 fw_rsv); 10257 10258 if (strlen(resp->active_pkg_name)) { 10259 int fw_ver_len = strlen(bp->fw_ver_str); 10260 10261 snprintf(bp->fw_ver_str + fw_ver_len, 10262 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", 10263 resp->active_pkg_name); 10264 bp->fw_cap |= BNXT_FW_CAP_PKG_VER; 10265 } 10266 10267 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout); 10268 if (!bp->hwrm_cmd_timeout) 10269 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 10270 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000; 10271 if (!bp->hwrm_cmd_max_timeout) 10272 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT; 10273 max_tmo_secs = bp->hwrm_cmd_max_timeout / 1000; 10274#ifdef CONFIG_DETECT_HUNG_TASK 10275 if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT || 10276 max_tmo_secs > CONFIG_DEFAULT_HUNG_TASK_TIMEOUT) { 10277 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog (kernel default %ds)\n", 10278 max_tmo_secs, CONFIG_DEFAULT_HUNG_TASK_TIMEOUT); 10279 } 10280#endif 10281 10282 if (resp->hwrm_intf_maj_8b >= 1) { 10283 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len); 10284 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len); 10285 } 10286 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN) 10287 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN; 10288 10289 bp->chip_num = le16_to_cpu(resp->chip_num); 10290 bp->chip_rev = resp->chip_rev; 10291 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev && 10292 !resp->chip_metal) 10293 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0; 10294 10295 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg); 10296 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && 10297 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) 10298 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD; 10299 10300 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) 10301 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; 10302 10303 if (dev_caps_cfg & 10304 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) 10305 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; 10306 10307 if (dev_caps_cfg & 10308 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) 10309 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; 10310 10311 if (dev_caps_cfg & 10312 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) 10313 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; 10314 10315hwrm_ver_get_exit: 10316 hwrm_req_drop(bp, req); 10317 return rc; 10318} 10319 10320int bnxt_hwrm_fw_set_time(struct bnxt *bp) 10321{ 10322 struct hwrm_fw_set_time_input *req; 10323 struct tm tm; 10324 time64_t now = ktime_get_real_seconds(); 10325 int rc; 10326 10327 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) || 10328 bp->hwrm_spec_code < 0x10400) 10329 return -EOPNOTSUPP; 10330 10331 time64_to_tm(now, 0, &tm); 10332 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME); 10333 if (rc) 10334 return rc; 10335 10336 req->year = cpu_to_le16(1900 + tm.tm_year); 10337 req->month = 1 + tm.tm_mon; 10338 req->day = tm.tm_mday; 10339 req->hour = tm.tm_hour; 10340 req->minute = tm.tm_min; 10341 req->second = tm.tm_sec; 10342 return hwrm_req_send(bp, req); 10343} 10344 10345static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask) 10346{ 10347 u64 sw_tmp; 10348 10349 hw &= mask; 10350 sw_tmp = (*sw & ~mask) | hw; 10351 if (hw < (*sw & mask)) 10352 sw_tmp += mask + 1; 10353 WRITE_ONCE(*sw, sw_tmp); 10354} 10355 10356static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, 10357 int count, bool ignore_zero) 10358{ 10359 int i; 10360 10361 for (i = 0; i < count; i++) { 10362 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); 10363 10364 if (ignore_zero && !hw) 10365 continue; 10366 10367 if (masks[i] == -1ULL) 10368 sw_stats[i] = hw; 10369 else 10370 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]); 10371 } 10372} 10373 10374static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats) 10375{ 10376 if (!stats->hw_stats) 10377 return; 10378 10379 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 10380 stats->hw_masks, stats->len / 8, false); 10381} 10382 10383static void bnxt_accumulate_all_stats(struct bnxt *bp) 10384{ 10385 struct bnxt_stats_mem *ring0_stats; 10386 bool ignore_zero = false; 10387 int i; 10388 10389 /* Chip bug. Counter intermittently becomes 0. */ 10390 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10391 ignore_zero = true; 10392 10393 for (i = 0; i < bp->cp_nr_rings; i++) { 10394 struct bnxt_napi *bnapi = bp->bnapi[i]; 10395 struct bnxt_cp_ring_info *cpr; 10396 struct bnxt_stats_mem *stats; 10397 10398 cpr = &bnapi->cp_ring; 10399 stats = &cpr->stats; 10400 if (!i) 10401 ring0_stats = stats; 10402 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats, 10403 ring0_stats->hw_masks, 10404 ring0_stats->len / 8, ignore_zero); 10405 } 10406 if (bp->flags & BNXT_FLAG_PORT_STATS) { 10407 struct bnxt_stats_mem *stats = &bp->port_stats; 10408 __le64 *hw_stats = stats->hw_stats; 10409 u64 *sw_stats = stats->sw_stats; 10410 u64 *masks = stats->hw_masks; 10411 int cnt; 10412 10413 cnt = sizeof(struct rx_port_stats) / 8; 10414 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 10415 10416 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10417 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10418 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 10419 cnt = sizeof(struct tx_port_stats) / 8; 10420 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false); 10421 } 10422 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) { 10423 bnxt_accumulate_stats(&bp->rx_port_stats_ext); 10424 bnxt_accumulate_stats(&bp->tx_port_stats_ext); 10425 } 10426} 10427 10428static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags) 10429{ 10430 struct hwrm_port_qstats_input *req; 10431 struct bnxt_pf_info *pf = &bp->pf; 10432 int rc; 10433 10434 if (!(bp->flags & BNXT_FLAG_PORT_STATS)) 10435 return 0; 10436 10437 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 10438 return -EOPNOTSUPP; 10439 10440 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS); 10441 if (rc) 10442 return rc; 10443 10444 req->flags = flags; 10445 req->port_id = cpu_to_le16(pf->port_id); 10446 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map + 10447 BNXT_TX_PORT_STATS_BYTE_OFFSET); 10448 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map); 10449 return hwrm_req_send(bp, req); 10450} 10451 10452static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags) 10453{ 10454 struct hwrm_queue_pri2cos_qcfg_output *resp_qc; 10455 struct hwrm_queue_pri2cos_qcfg_input *req_qc; 10456 struct hwrm_port_qstats_ext_output *resp_qs; 10457 struct hwrm_port_qstats_ext_input *req_qs; 10458 struct bnxt_pf_info *pf = &bp->pf; 10459 u32 tx_stat_size; 10460 int rc; 10461 10462 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT)) 10463 return 0; 10464 10465 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)) 10466 return -EOPNOTSUPP; 10467 10468 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT); 10469 if (rc) 10470 return rc; 10471 10472 req_qs->flags = flags; 10473 req_qs->port_id = cpu_to_le16(pf->port_id); 10474 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); 10475 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map); 10476 tx_stat_size = bp->tx_port_stats_ext.hw_stats ? 10477 sizeof(struct tx_port_stats_ext) : 0; 10478 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); 10479 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map); 10480 resp_qs = hwrm_req_hold(bp, req_qs); 10481 rc = hwrm_req_send(bp, req_qs); 10482 if (!rc) { 10483 bp->fw_rx_stats_ext_size = 10484 le16_to_cpu(resp_qs->rx_stat_size) / 8; 10485 if (BNXT_FW_MAJ(bp) < 220 && 10486 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) 10487 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; 10488 10489 bp->fw_tx_stats_ext_size = tx_stat_size ? 10490 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; 10491 } else { 10492 bp->fw_rx_stats_ext_size = 0; 10493 bp->fw_tx_stats_ext_size = 0; 10494 } 10495 hwrm_req_drop(bp, req_qs); 10496 10497 if (flags) 10498 return rc; 10499 10500 if (bp->fw_tx_stats_ext_size <= 10501 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { 10502 bp->pri2cos_valid = 0; 10503 return rc; 10504 } 10505 10506 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG); 10507 if (rc) 10508 return rc; 10509 10510 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); 10511 10512 resp_qc = hwrm_req_hold(bp, req_qc); 10513 rc = hwrm_req_send(bp, req_qc); 10514 if (!rc) { 10515 u8 *pri2cos; 10516 int i, j; 10517 10518 pri2cos = &resp_qc->pri0_cos_queue_id; 10519 for (i = 0; i < 8; i++) { 10520 u8 queue_id = pri2cos[i]; 10521 u8 queue_idx; 10522 10523 /* Per port queue IDs start from 0, 10, 20, etc */ 10524 queue_idx = queue_id % 10; 10525 if (queue_idx > BNXT_MAX_QUEUE) { 10526 bp->pri2cos_valid = false; 10527 hwrm_req_drop(bp, req_qc); 10528 return rc; 10529 } 10530 for (j = 0; j < bp->max_q; j++) { 10531 if (bp->q_ids[j] == queue_id) 10532 bp->pri2cos_idx[i] = queue_idx; 10533 } 10534 } 10535 bp->pri2cos_valid = true; 10536 } 10537 hwrm_req_drop(bp, req_qc); 10538 10539 return rc; 10540} 10541 10542static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp) 10543{ 10544 bnxt_hwrm_tunnel_dst_port_free(bp, 10545 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN); 10546 bnxt_hwrm_tunnel_dst_port_free(bp, 10547 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE); 10548} 10549 10550static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa) 10551{ 10552 int rc, i; 10553 u32 tpa_flags = 0; 10554 10555 if (set_tpa) 10556 tpa_flags = bp->flags & BNXT_FLAG_TPA; 10557 else if (BNXT_NO_FW_ACCESS(bp)) 10558 return 0; 10559 for (i = 0; i < bp->nr_vnics; i++) { 10560 rc = bnxt_hwrm_vnic_set_tpa(bp, &bp->vnic_info[i], tpa_flags); 10561 if (rc) { 10562 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n", 10563 i, rc); 10564 return rc; 10565 } 10566 } 10567 return 0; 10568} 10569 10570static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp) 10571{ 10572 int i; 10573 10574 for (i = 0; i < bp->nr_vnics; i++) 10575 bnxt_hwrm_vnic_set_rss(bp, &bp->vnic_info[i], false); 10576} 10577 10578static void bnxt_clear_vnic(struct bnxt *bp) 10579{ 10580 if (!bp->vnic_info) 10581 return; 10582 10583 bnxt_hwrm_clear_vnic_filter(bp); 10584 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) { 10585 /* clear all RSS setting before free vnic ctx */ 10586 bnxt_hwrm_clear_vnic_rss(bp); 10587 bnxt_hwrm_vnic_ctx_free(bp); 10588 } 10589 /* before free the vnic, undo the vnic tpa settings */ 10590 if (bp->flags & BNXT_FLAG_TPA) 10591 bnxt_set_tpa(bp, false); 10592 bnxt_hwrm_vnic_free(bp); 10593 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10594 bnxt_hwrm_vnic_ctx_free(bp); 10595} 10596 10597static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path, 10598 bool irq_re_init) 10599{ 10600 bnxt_clear_vnic(bp); 10601 bnxt_hwrm_ring_free(bp, close_path); 10602 bnxt_hwrm_ring_grp_free(bp); 10603 if (irq_re_init) { 10604 bnxt_hwrm_stat_ctx_free(bp); 10605 bnxt_hwrm_free_tunnel_ports(bp); 10606 } 10607} 10608 10609static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode) 10610{ 10611 struct hwrm_func_cfg_input *req; 10612 u8 evb_mode; 10613 int rc; 10614 10615 if (br_mode == BRIDGE_MODE_VEB) 10616 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB; 10617 else if (br_mode == BRIDGE_MODE_VEPA) 10618 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA; 10619 else 10620 return -EINVAL; 10621 10622 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 10623 if (rc) 10624 return rc; 10625 10626 req->fid = cpu_to_le16(0xffff); 10627 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE); 10628 req->evb_mode = evb_mode; 10629 return hwrm_req_send(bp, req); 10630} 10631 10632static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size) 10633{ 10634 struct hwrm_func_cfg_input *req; 10635 int rc; 10636 10637 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803) 10638 return 0; 10639 10640 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req); 10641 if (rc) 10642 return rc; 10643 10644 req->fid = cpu_to_le16(0xffff); 10645 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE); 10646 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64; 10647 if (size == 128) 10648 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128; 10649 10650 return hwrm_req_send(bp, req); 10651} 10652 10653static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic) 10654{ 10655 int rc; 10656 10657 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) 10658 goto skip_rss_ctx; 10659 10660 /* allocate context for vnic */ 10661 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 0); 10662 if (rc) { 10663 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 10664 vnic->vnic_id, rc); 10665 goto vnic_setup_err; 10666 } 10667 bp->rsscos_nr_ctxs++; 10668 10669 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 10670 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 1); 10671 if (rc) { 10672 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n", 10673 vnic->vnic_id, rc); 10674 goto vnic_setup_err; 10675 } 10676 bp->rsscos_nr_ctxs++; 10677 } 10678 10679skip_rss_ctx: 10680 /* configure default vnic, ring grp */ 10681 rc = bnxt_hwrm_vnic_cfg(bp, vnic); 10682 if (rc) { 10683 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 10684 vnic->vnic_id, rc); 10685 goto vnic_setup_err; 10686 } 10687 10688 /* Enable RSS hashing on vnic */ 10689 rc = bnxt_hwrm_vnic_set_rss(bp, vnic, true); 10690 if (rc) { 10691 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n", 10692 vnic->vnic_id, rc); 10693 goto vnic_setup_err; 10694 } 10695 10696 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 10697 rc = bnxt_hwrm_vnic_set_hds(bp, vnic); 10698 if (rc) { 10699 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 10700 vnic->vnic_id, rc); 10701 } 10702 } 10703 10704vnic_setup_err: 10705 return rc; 10706} 10707 10708int bnxt_hwrm_vnic_update(struct bnxt *bp, struct bnxt_vnic_info *vnic, 10709 u8 valid) 10710{ 10711 struct hwrm_vnic_update_input *req; 10712 int rc; 10713 10714 rc = hwrm_req_init(bp, req, HWRM_VNIC_UPDATE); 10715 if (rc) 10716 return rc; 10717 10718 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id); 10719 10720 if (valid & VNIC_UPDATE_REQ_ENABLES_MRU_VALID) 10721 req->mru = cpu_to_le16(vnic->mru); 10722 10723 req->enables = cpu_to_le32(valid); 10724 10725 return hwrm_req_send(bp, req); 10726} 10727 10728int bnxt_hwrm_vnic_rss_cfg_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic) 10729{ 10730 int rc; 10731 10732 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true); 10733 if (rc) { 10734 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 10735 vnic->vnic_id, rc); 10736 return rc; 10737 } 10738 rc = bnxt_hwrm_vnic_cfg(bp, vnic); 10739 if (rc) 10740 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n", 10741 vnic->vnic_id, rc); 10742 return rc; 10743} 10744 10745int __bnxt_setup_vnic_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic) 10746{ 10747 int rc, i, nr_ctxs; 10748 10749 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); 10750 for (i = 0; i < nr_ctxs; i++) { 10751 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, i); 10752 if (rc) { 10753 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n", 10754 vnic->vnic_id, i, rc); 10755 break; 10756 } 10757 bp->rsscos_nr_ctxs++; 10758 } 10759 if (i < nr_ctxs) 10760 return -ENOMEM; 10761 10762 rc = bnxt_hwrm_vnic_rss_cfg_p5(bp, vnic); 10763 if (rc) 10764 return rc; 10765 10766 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 10767 rc = bnxt_hwrm_vnic_set_hds(bp, vnic); 10768 if (rc) { 10769 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n", 10770 vnic->vnic_id, rc); 10771 } 10772 } 10773 return rc; 10774} 10775 10776static int bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic) 10777{ 10778 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10779 return __bnxt_setup_vnic_p5(bp, vnic); 10780 else 10781 return __bnxt_setup_vnic(bp, vnic); 10782} 10783 10784static int bnxt_alloc_and_setup_vnic(struct bnxt *bp, 10785 struct bnxt_vnic_info *vnic, 10786 u16 start_rx_ring_idx, int rx_rings) 10787{ 10788 int rc; 10789 10790 rc = bnxt_hwrm_vnic_alloc(bp, vnic, start_rx_ring_idx, rx_rings); 10791 if (rc) { 10792 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n", 10793 vnic->vnic_id, rc); 10794 return rc; 10795 } 10796 return bnxt_setup_vnic(bp, vnic); 10797} 10798 10799static int bnxt_alloc_rfs_vnics(struct bnxt *bp) 10800{ 10801 struct bnxt_vnic_info *vnic; 10802 int i, rc = 0; 10803 10804 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) { 10805 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE]; 10806 return bnxt_alloc_and_setup_vnic(bp, vnic, 0, bp->rx_nr_rings); 10807 } 10808 10809 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 10810 return 0; 10811 10812 for (i = 0; i < bp->rx_nr_rings; i++) { 10813 u16 vnic_id = i + 1; 10814 u16 ring_id = i; 10815 10816 if (vnic_id >= bp->nr_vnics) 10817 break; 10818 10819 vnic = &bp->vnic_info[vnic_id]; 10820 vnic->flags |= BNXT_VNIC_RFS_FLAG; 10821 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) 10822 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG; 10823 if (bnxt_alloc_and_setup_vnic(bp, &bp->vnic_info[vnic_id], ring_id, 1)) 10824 break; 10825 } 10826 return rc; 10827} 10828 10829void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx, 10830 bool all) 10831{ 10832 struct bnxt_vnic_info *vnic = &rss_ctx->vnic; 10833 struct bnxt_filter_base *usr_fltr, *tmp; 10834 struct bnxt_ntuple_filter *ntp_fltr; 10835 int i; 10836 10837 bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic); 10838 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) { 10839 if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID) 10840 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i); 10841 } 10842 if (!all) 10843 return; 10844 10845 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) { 10846 if ((usr_fltr->flags & BNXT_ACT_RSS_CTX) && 10847 usr_fltr->fw_vnic_id == rss_ctx->index) { 10848 ntp_fltr = container_of(usr_fltr, 10849 struct bnxt_ntuple_filter, 10850 base); 10851 bnxt_hwrm_cfa_ntuple_filter_free(bp, ntp_fltr); 10852 bnxt_del_ntp_filter(bp, ntp_fltr); 10853 bnxt_del_one_usr_fltr(bp, usr_fltr); 10854 } 10855 } 10856 10857 if (vnic->rss_table) 10858 dma_free_coherent(&bp->pdev->dev, vnic->rss_table_size, 10859 vnic->rss_table, 10860 vnic->rss_table_dma_addr); 10861 bp->num_rss_ctx--; 10862} 10863 10864static bool bnxt_vnic_has_rx_ring(struct bnxt *bp, struct bnxt_vnic_info *vnic, 10865 int rxr_id) 10866{ 10867 u16 tbl_size = bnxt_get_rxfh_indir_size(bp->dev); 10868 int i, vnic_rx; 10869 10870 /* Ntuple VNIC always has all the rx rings. Any change of ring id 10871 * must be updated because a future filter may use it. 10872 */ 10873 if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG) 10874 return true; 10875 10876 for (i = 0; i < tbl_size; i++) { 10877 if (vnic->flags & BNXT_VNIC_RSSCTX_FLAG) 10878 vnic_rx = ethtool_rxfh_context_indir(vnic->rss_ctx)[i]; 10879 else 10880 vnic_rx = bp->rss_indir_tbl[i]; 10881 10882 if (rxr_id == vnic_rx) 10883 return true; 10884 } 10885 10886 return false; 10887} 10888 10889static int bnxt_set_vnic_mru_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic, 10890 u16 mru, int rxr_id) 10891{ 10892 int rc; 10893 10894 if (!bnxt_vnic_has_rx_ring(bp, vnic, rxr_id)) 10895 return 0; 10896 10897 if (mru) { 10898 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true); 10899 if (rc) { 10900 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n", 10901 vnic->vnic_id, rc); 10902 return rc; 10903 } 10904 } 10905 vnic->mru = mru; 10906 bnxt_hwrm_vnic_update(bp, vnic, 10907 VNIC_UPDATE_REQ_ENABLES_MRU_VALID); 10908 10909 return 0; 10910} 10911 10912static int bnxt_set_rss_ctx_vnic_mru(struct bnxt *bp, u16 mru, int rxr_id) 10913{ 10914 struct ethtool_rxfh_context *ctx; 10915 unsigned long context; 10916 int rc; 10917 10918 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) { 10919 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx); 10920 struct bnxt_vnic_info *vnic = &rss_ctx->vnic; 10921 10922 rc = bnxt_set_vnic_mru_p5(bp, vnic, mru, rxr_id); 10923 if (rc) 10924 return rc; 10925 } 10926 10927 return 0; 10928} 10929 10930static void bnxt_hwrm_realloc_rss_ctx_vnic(struct bnxt *bp) 10931{ 10932 bool set_tpa = !!(bp->flags & BNXT_FLAG_TPA); 10933 struct ethtool_rxfh_context *ctx; 10934 unsigned long context; 10935 10936 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) { 10937 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx); 10938 struct bnxt_vnic_info *vnic = &rss_ctx->vnic; 10939 10940 if (bnxt_hwrm_vnic_alloc(bp, vnic, 0, bp->rx_nr_rings) || 10941 bnxt_hwrm_vnic_set_tpa(bp, vnic, set_tpa) || 10942 __bnxt_setup_vnic_p5(bp, vnic)) { 10943 netdev_err(bp->dev, "Failed to restore RSS ctx %d\n", 10944 rss_ctx->index); 10945 bnxt_del_one_rss_ctx(bp, rss_ctx, true); 10946 ethtool_rxfh_context_lost(bp->dev, rss_ctx->index); 10947 } 10948 } 10949} 10950 10951static void bnxt_clear_rss_ctxs(struct bnxt *bp) 10952{ 10953 struct ethtool_rxfh_context *ctx; 10954 unsigned long context; 10955 10956 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) { 10957 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx); 10958 10959 bnxt_del_one_rss_ctx(bp, rss_ctx, false); 10960 } 10961} 10962 10963/* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */ 10964static bool bnxt_promisc_ok(struct bnxt *bp) 10965{ 10966#ifdef CONFIG_BNXT_SRIOV 10967 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf)) 10968 return false; 10969#endif 10970 return true; 10971} 10972 10973static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) 10974{ 10975 struct bnxt_vnic_info *vnic = &bp->vnic_info[1]; 10976 unsigned int rc = 0; 10977 10978 rc = bnxt_hwrm_vnic_alloc(bp, vnic, bp->rx_nr_rings - 1, 1); 10979 if (rc) { 10980 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 10981 rc); 10982 return rc; 10983 } 10984 10985 rc = bnxt_hwrm_vnic_cfg(bp, vnic); 10986 if (rc) { 10987 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", 10988 rc); 10989 return rc; 10990 } 10991 return rc; 10992} 10993 10994static int bnxt_cfg_rx_mode(struct bnxt *); 10995static bool bnxt_mc_list_updated(struct bnxt *, u32 *); 10996 10997static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init) 10998{ 10999 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 11000 int rc = 0; 11001 unsigned int rx_nr_rings = bp->rx_nr_rings; 11002 11003 if (irq_re_init) { 11004 rc = bnxt_hwrm_stat_ctx_alloc(bp); 11005 if (rc) { 11006 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n", 11007 rc); 11008 goto err_out; 11009 } 11010 } 11011 11012 rc = bnxt_hwrm_ring_alloc(bp); 11013 if (rc) { 11014 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc); 11015 goto err_out; 11016 } 11017 11018 rc = bnxt_hwrm_ring_grp_alloc(bp); 11019 if (rc) { 11020 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc); 11021 goto err_out; 11022 } 11023 11024 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 11025 rx_nr_rings--; 11026 11027 /* default vnic 0 */ 11028 rc = bnxt_hwrm_vnic_alloc(bp, vnic, 0, rx_nr_rings); 11029 if (rc) { 11030 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc); 11031 goto err_out; 11032 } 11033 11034 if (BNXT_VF(bp)) 11035 bnxt_hwrm_func_qcfg(bp); 11036 11037 rc = bnxt_setup_vnic(bp, vnic); 11038 if (rc) 11039 goto err_out; 11040 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA) 11041 bnxt_hwrm_update_rss_hash_cfg(bp); 11042 11043 if (bp->flags & BNXT_FLAG_RFS) { 11044 rc = bnxt_alloc_rfs_vnics(bp); 11045 if (rc) 11046 goto err_out; 11047 } 11048 11049 if (bp->flags & BNXT_FLAG_TPA) { 11050 rc = bnxt_set_tpa(bp, true); 11051 if (rc) 11052 goto err_out; 11053 } 11054 11055 if (BNXT_VF(bp)) 11056 bnxt_update_vf_mac(bp); 11057 11058 /* Filter for default vnic 0 */ 11059 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr); 11060 if (rc) { 11061 if (BNXT_VF(bp) && rc == -ENODEV) 11062 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n"); 11063 else 11064 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 11065 goto err_out; 11066 } 11067 vnic->uc_filter_count = 1; 11068 11069 vnic->rx_mask = 0; 11070 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state)) 11071 goto skip_rx_mask; 11072 11073 if (bp->dev->flags & IFF_BROADCAST) 11074 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 11075 11076 if (bp->dev->flags & IFF_PROMISC) 11077 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 11078 11079 if (bp->dev->flags & IFF_ALLMULTI) { 11080 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 11081 vnic->mc_list_count = 0; 11082 } else if (bp->dev->flags & IFF_MULTICAST) { 11083 u32 mask = 0; 11084 11085 bnxt_mc_list_updated(bp, &mask); 11086 vnic->rx_mask |= mask; 11087 } 11088 11089 rc = bnxt_cfg_rx_mode(bp); 11090 if (rc) 11091 goto err_out; 11092 11093skip_rx_mask: 11094 rc = bnxt_hwrm_set_coal(bp); 11095 if (rc) 11096 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n", 11097 rc); 11098 11099 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 11100 rc = bnxt_setup_nitroa0_vnic(bp); 11101 if (rc) 11102 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n", 11103 rc); 11104 } 11105 11106 if (BNXT_VF(bp)) { 11107 bnxt_hwrm_func_qcfg(bp); 11108 netdev_update_features(bp->dev); 11109 } 11110 11111 return 0; 11112 11113err_out: 11114 bnxt_hwrm_resource_free(bp, 0, true); 11115 11116 return rc; 11117} 11118 11119static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) 11120{ 11121 bnxt_hwrm_resource_free(bp, 1, irq_re_init); 11122 return 0; 11123} 11124 11125static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) 11126{ 11127 bnxt_init_cp_rings(bp); 11128 bnxt_init_rx_rings(bp); 11129 bnxt_init_tx_rings(bp); 11130 bnxt_init_ring_grps(bp, irq_re_init); 11131 bnxt_init_vnics(bp); 11132 11133 return bnxt_init_chip(bp, irq_re_init); 11134} 11135 11136static int bnxt_set_real_num_queues(struct bnxt *bp) 11137{ 11138 int rc; 11139 struct net_device *dev = bp->dev; 11140 11141 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings - 11142 bp->tx_nr_rings_xdp); 11143 if (rc) 11144 return rc; 11145 11146 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings); 11147 if (rc) 11148 return rc; 11149 11150#ifdef CONFIG_RFS_ACCEL 11151 if (bp->flags & BNXT_FLAG_RFS) 11152 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings); 11153#endif 11154 11155 return rc; 11156} 11157 11158static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 11159 bool shared) 11160{ 11161 int _rx = *rx, _tx = *tx; 11162 11163 if (shared) { 11164 *rx = min_t(int, _rx, max); 11165 *tx = min_t(int, _tx, max); 11166 } else { 11167 if (max < 2) 11168 return -ENOMEM; 11169 11170 while (_rx + _tx > max) { 11171 if (_rx > _tx && _rx > 1) 11172 _rx--; 11173 else if (_tx > 1) 11174 _tx--; 11175 } 11176 *rx = _rx; 11177 *tx = _tx; 11178 } 11179 return 0; 11180} 11181 11182static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp) 11183{ 11184 return (tx - tx_xdp) / tx_sets + tx_xdp; 11185} 11186 11187int bnxt_num_tx_to_cp(struct bnxt *bp, int tx) 11188{ 11189 int tcs = bp->num_tc; 11190 11191 if (!tcs) 11192 tcs = 1; 11193 return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp); 11194} 11195 11196static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp) 11197{ 11198 int tcs = bp->num_tc; 11199 11200 return (tx_cp - bp->tx_nr_rings_xdp) * tcs + 11201 bp->tx_nr_rings_xdp; 11202} 11203 11204static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max, 11205 bool sh) 11206{ 11207 int tx_cp = bnxt_num_tx_to_cp(bp, *tx); 11208 11209 if (tx_cp != *tx) { 11210 int tx_saved = tx_cp, rc; 11211 11212 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh); 11213 if (rc) 11214 return rc; 11215 if (tx_cp != tx_saved) 11216 *tx = bnxt_num_cp_to_tx(bp, tx_cp); 11217 return 0; 11218 } 11219 return __bnxt_trim_rings(bp, rx, tx, max, sh); 11220} 11221 11222static void bnxt_setup_msix(struct bnxt *bp) 11223{ 11224 const int len = sizeof(bp->irq_tbl[0].name); 11225 struct net_device *dev = bp->dev; 11226 int tcs, i; 11227 11228 tcs = bp->num_tc; 11229 if (tcs) { 11230 int i, off, count; 11231 11232 for (i = 0; i < tcs; i++) { 11233 count = bp->tx_nr_rings_per_tc; 11234 off = BNXT_TC_TO_RING_BASE(bp, i); 11235 netdev_set_tc_queue(dev, i, count, off); 11236 } 11237 } 11238 11239 for (i = 0; i < bp->cp_nr_rings; i++) { 11240 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 11241 char *attr; 11242 11243 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 11244 attr = "TxRx"; 11245 else if (i < bp->rx_nr_rings) 11246 attr = "rx"; 11247 else 11248 attr = "tx"; 11249 11250 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name, 11251 attr, i); 11252 bp->irq_tbl[map_idx].handler = bnxt_msix; 11253 } 11254} 11255 11256static int bnxt_init_int_mode(struct bnxt *bp); 11257 11258static int bnxt_change_msix(struct bnxt *bp, int total) 11259{ 11260 struct msi_map map; 11261 int i; 11262 11263 /* add MSIX to the end if needed */ 11264 for (i = bp->total_irqs; i < total; i++) { 11265 map = pci_msix_alloc_irq_at(bp->pdev, i, NULL); 11266 if (map.index < 0) 11267 return bp->total_irqs; 11268 bp->irq_tbl[i].vector = map.virq; 11269 bp->total_irqs++; 11270 } 11271 11272 /* trim MSIX from the end if needed */ 11273 for (i = bp->total_irqs; i > total; i--) { 11274 map.index = i - 1; 11275 map.virq = bp->irq_tbl[i - 1].vector; 11276 pci_msix_free_irq(bp->pdev, map); 11277 bp->total_irqs--; 11278 } 11279 return bp->total_irqs; 11280} 11281 11282static int bnxt_setup_int_mode(struct bnxt *bp) 11283{ 11284 int rc; 11285 11286 if (!bp->irq_tbl) { 11287 rc = bnxt_init_int_mode(bp); 11288 if (rc || !bp->irq_tbl) 11289 return rc ?: -ENODEV; 11290 } 11291 11292 bnxt_setup_msix(bp); 11293 11294 rc = bnxt_set_real_num_queues(bp); 11295 return rc; 11296} 11297 11298static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp) 11299{ 11300 return bp->hw_resc.max_rsscos_ctxs; 11301} 11302 11303static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp) 11304{ 11305 return bp->hw_resc.max_vnics; 11306} 11307 11308unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp) 11309{ 11310 return bp->hw_resc.max_stat_ctxs; 11311} 11312 11313unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp) 11314{ 11315 return bp->hw_resc.max_cp_rings; 11316} 11317 11318static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp) 11319{ 11320 unsigned int cp = bp->hw_resc.max_cp_rings; 11321 11322 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 11323 cp -= bnxt_get_ulp_msix_num(bp); 11324 11325 return cp; 11326} 11327 11328static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp) 11329{ 11330 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 11331 11332 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 11333 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs); 11334 11335 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings); 11336} 11337 11338static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs) 11339{ 11340 bp->hw_resc.max_irqs = max_irqs; 11341} 11342 11343unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) 11344{ 11345 unsigned int cp; 11346 11347 cp = bnxt_get_max_func_cp_rings_for_en(bp); 11348 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 11349 return cp - bp->rx_nr_rings - bp->tx_nr_rings; 11350 else 11351 return cp - bp->cp_nr_rings; 11352} 11353 11354unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) 11355{ 11356 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); 11357} 11358 11359static int bnxt_get_avail_msix(struct bnxt *bp, int num) 11360{ 11361 int max_irq = bnxt_get_max_func_irqs(bp); 11362 int total_req = bp->cp_nr_rings + num; 11363 11364 if (max_irq < total_req) { 11365 num = max_irq - bp->cp_nr_rings; 11366 if (num <= 0) 11367 return 0; 11368 } 11369 return num; 11370} 11371 11372static int bnxt_get_num_msix(struct bnxt *bp) 11373{ 11374 if (!BNXT_NEW_RM(bp)) 11375 return bnxt_get_max_func_irqs(bp); 11376 11377 return bnxt_nq_rings_in_use(bp); 11378} 11379 11380static int bnxt_init_int_mode(struct bnxt *bp) 11381{ 11382 int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp, tbl_size; 11383 11384 total_vecs = bnxt_get_num_msix(bp); 11385 max = bnxt_get_max_func_irqs(bp); 11386 if (total_vecs > max) 11387 total_vecs = max; 11388 11389 if (!total_vecs) 11390 return 0; 11391 11392 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) 11393 min = 2; 11394 11395 total_vecs = pci_alloc_irq_vectors(bp->pdev, min, total_vecs, 11396 PCI_IRQ_MSIX); 11397 ulp_msix = bnxt_get_ulp_msix_num(bp); 11398 if (total_vecs < 0 || total_vecs < ulp_msix) { 11399 rc = -ENODEV; 11400 goto msix_setup_exit; 11401 } 11402 11403 tbl_size = total_vecs; 11404 if (pci_msix_can_alloc_dyn(bp->pdev)) 11405 tbl_size = max; 11406 bp->irq_tbl = kcalloc(tbl_size, sizeof(*bp->irq_tbl), GFP_KERNEL); 11407 if (bp->irq_tbl) { 11408 for (i = 0; i < total_vecs; i++) 11409 bp->irq_tbl[i].vector = pci_irq_vector(bp->pdev, i); 11410 11411 bp->total_irqs = total_vecs; 11412 /* Trim rings based upon num of vectors allocated */ 11413 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings, 11414 total_vecs - ulp_msix, min == 1); 11415 if (rc) 11416 goto msix_setup_exit; 11417 11418 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); 11419 bp->cp_nr_rings = (min == 1) ? 11420 max_t(int, tx_cp, bp->rx_nr_rings) : 11421 tx_cp + bp->rx_nr_rings; 11422 11423 } else { 11424 rc = -ENOMEM; 11425 goto msix_setup_exit; 11426 } 11427 return 0; 11428 11429msix_setup_exit: 11430 netdev_err(bp->dev, "bnxt_init_int_mode err: %x\n", rc); 11431 kfree(bp->irq_tbl); 11432 bp->irq_tbl = NULL; 11433 pci_free_irq_vectors(bp->pdev); 11434 return rc; 11435} 11436 11437static void bnxt_clear_int_mode(struct bnxt *bp) 11438{ 11439 pci_free_irq_vectors(bp->pdev); 11440 11441 kfree(bp->irq_tbl); 11442 bp->irq_tbl = NULL; 11443} 11444 11445int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) 11446{ 11447 bool irq_cleared = false; 11448 bool irq_change = false; 11449 int tcs = bp->num_tc; 11450 int irqs_required; 11451 int rc; 11452 11453 if (!bnxt_need_reserve_rings(bp)) 11454 return 0; 11455 11456 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) { 11457 int ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want); 11458 11459 if (ulp_msix > bp->ulp_num_msix_want) 11460 ulp_msix = bp->ulp_num_msix_want; 11461 irqs_required = ulp_msix + bp->cp_nr_rings; 11462 } else { 11463 irqs_required = bnxt_get_num_msix(bp); 11464 } 11465 11466 if (irq_re_init && BNXT_NEW_RM(bp) && irqs_required != bp->total_irqs) { 11467 irq_change = true; 11468 if (!pci_msix_can_alloc_dyn(bp->pdev)) { 11469 bnxt_ulp_irq_stop(bp); 11470 bnxt_clear_int_mode(bp); 11471 irq_cleared = true; 11472 } 11473 } 11474 rc = __bnxt_reserve_rings(bp); 11475 if (irq_cleared) { 11476 if (!rc) 11477 rc = bnxt_init_int_mode(bp); 11478 bnxt_ulp_irq_restart(bp, rc); 11479 } else if (irq_change && !rc) { 11480 if (bnxt_change_msix(bp, irqs_required) != irqs_required) 11481 rc = -ENOSPC; 11482 } 11483 if (rc) { 11484 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc); 11485 return rc; 11486 } 11487 if (tcs && (bp->tx_nr_rings_per_tc * tcs != 11488 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) { 11489 netdev_err(bp->dev, "tx ring reservation failure\n"); 11490 netdev_reset_tc(bp->dev); 11491 bp->num_tc = 0; 11492 if (bp->tx_nr_rings_xdp) 11493 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp; 11494 else 11495 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 11496 return -ENOMEM; 11497 } 11498 return 0; 11499} 11500 11501static void bnxt_tx_queue_stop(struct bnxt *bp, int idx) 11502{ 11503 struct bnxt_tx_ring_info *txr; 11504 struct netdev_queue *txq; 11505 struct bnxt_napi *bnapi; 11506 int i; 11507 11508 bnapi = bp->bnapi[idx]; 11509 bnxt_for_each_napi_tx(i, bnapi, txr) { 11510 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 11511 synchronize_net(); 11512 11513 if (!(bnapi->flags & BNXT_NAPI_FLAG_XDP)) { 11514 txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 11515 if (txq) { 11516 __netif_tx_lock_bh(txq); 11517 netif_tx_stop_queue(txq); 11518 __netif_tx_unlock_bh(txq); 11519 } 11520 } 11521 11522 if (!bp->tph_mode) 11523 continue; 11524 11525 bnxt_hwrm_tx_ring_free(bp, txr, true); 11526 bnxt_hwrm_cp_ring_free(bp, txr->tx_cpr); 11527 bnxt_free_one_tx_ring_skbs(bp, txr, txr->txq_index); 11528 bnxt_clear_one_cp_ring(bp, txr->tx_cpr); 11529 } 11530} 11531 11532static int bnxt_tx_queue_start(struct bnxt *bp, int idx) 11533{ 11534 struct bnxt_tx_ring_info *txr; 11535 struct netdev_queue *txq; 11536 struct bnxt_napi *bnapi; 11537 int rc, i; 11538 11539 bnapi = bp->bnapi[idx]; 11540 /* All rings have been reserved and previously allocated. 11541 * Reallocating with the same parameters should never fail. 11542 */ 11543 bnxt_for_each_napi_tx(i, bnapi, txr) { 11544 if (!bp->tph_mode) 11545 goto start_tx; 11546 11547 rc = bnxt_hwrm_cp_ring_alloc_p5(bp, txr->tx_cpr); 11548 if (rc) 11549 return rc; 11550 11551 rc = bnxt_hwrm_tx_ring_alloc(bp, txr, false); 11552 if (rc) 11553 return rc; 11554 11555 txr->tx_prod = 0; 11556 txr->tx_cons = 0; 11557 txr->tx_hw_cons = 0; 11558start_tx: 11559 WRITE_ONCE(txr->dev_state, 0); 11560 synchronize_net(); 11561 11562 if (bnapi->flags & BNXT_NAPI_FLAG_XDP) 11563 continue; 11564 11565 txq = netdev_get_tx_queue(bp->dev, txr->txq_index); 11566 if (txq) 11567 netif_tx_start_queue(txq); 11568 } 11569 11570 return 0; 11571} 11572 11573static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, 11574 const cpumask_t *mask) 11575{ 11576 struct bnxt_irq *irq; 11577 u16 tag; 11578 int err; 11579 11580 irq = container_of(notify, struct bnxt_irq, affinity_notify); 11581 11582 if (!irq->bp->tph_mode) 11583 return; 11584 11585 cpumask_copy(irq->cpu_mask, mask); 11586 11587 if (irq->ring_nr >= irq->bp->rx_nr_rings) 11588 return; 11589 11590 if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, 11591 cpumask_first(irq->cpu_mask), &tag)) 11592 return; 11593 11594 if (pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag)) 11595 return; 11596 11597 netdev_lock(irq->bp->dev); 11598 if (netif_running(irq->bp->dev)) { 11599 err = netdev_rx_queue_restart(irq->bp->dev, irq->ring_nr); 11600 if (err) 11601 netdev_err(irq->bp->dev, 11602 "RX queue restart failed: err=%d\n", err); 11603 } 11604 netdev_unlock(irq->bp->dev); 11605} 11606 11607static void bnxt_irq_affinity_release(struct kref *ref) 11608{ 11609 struct irq_affinity_notify *notify = 11610 container_of(ref, struct irq_affinity_notify, kref); 11611 struct bnxt_irq *irq; 11612 11613 irq = container_of(notify, struct bnxt_irq, affinity_notify); 11614 11615 if (!irq->bp->tph_mode) 11616 return; 11617 11618 if (pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, 0)) { 11619 netdev_err(irq->bp->dev, 11620 "Setting ST=0 for MSIX entry %d failed\n", 11621 irq->msix_nr); 11622 return; 11623 } 11624} 11625 11626static void bnxt_release_irq_notifier(struct bnxt_irq *irq) 11627{ 11628 irq_set_affinity_notifier(irq->vector, NULL); 11629} 11630 11631static void bnxt_register_irq_notifier(struct bnxt *bp, struct bnxt_irq *irq) 11632{ 11633 struct irq_affinity_notify *notify; 11634 11635 irq->bp = bp; 11636 11637 /* Nothing to do if TPH is not enabled */ 11638 if (!bp->tph_mode) 11639 return; 11640 11641 /* Register IRQ affinity notifier */ 11642 notify = &irq->affinity_notify; 11643 notify->irq = irq->vector; 11644 notify->notify = bnxt_irq_affinity_notify; 11645 notify->release = bnxt_irq_affinity_release; 11646 11647 irq_set_affinity_notifier(irq->vector, notify); 11648} 11649 11650static void bnxt_free_irq(struct bnxt *bp) 11651{ 11652 struct bnxt_irq *irq; 11653 int i; 11654 11655#ifdef CONFIG_RFS_ACCEL 11656 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); 11657 bp->dev->rx_cpu_rmap = NULL; 11658#endif 11659 if (!bp->irq_tbl || !bp->bnapi) 11660 return; 11661 11662 for (i = 0; i < bp->cp_nr_rings; i++) { 11663 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 11664 11665 irq = &bp->irq_tbl[map_idx]; 11666 if (irq->requested) { 11667 if (irq->have_cpumask) { 11668 irq_update_affinity_hint(irq->vector, NULL); 11669 free_cpumask_var(irq->cpu_mask); 11670 irq->have_cpumask = 0; 11671 } 11672 11673 bnxt_release_irq_notifier(irq); 11674 11675 free_irq(irq->vector, bp->bnapi[i]); 11676 } 11677 11678 irq->requested = 0; 11679 } 11680 11681 /* Disable TPH support */ 11682 pcie_disable_tph(bp->pdev); 11683 bp->tph_mode = 0; 11684} 11685 11686static int bnxt_request_irq(struct bnxt *bp) 11687{ 11688 struct cpu_rmap *rmap = NULL; 11689 int i, j, rc = 0; 11690 unsigned long flags = 0; 11691 11692 rc = bnxt_setup_int_mode(bp); 11693 if (rc) { 11694 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n", 11695 rc); 11696 return rc; 11697 } 11698#ifdef CONFIG_RFS_ACCEL 11699 rmap = bp->dev->rx_cpu_rmap; 11700#endif 11701 11702 /* Enable TPH support as part of IRQ request */ 11703 rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); 11704 if (!rc) 11705 bp->tph_mode = PCI_TPH_ST_IV_MODE; 11706 11707 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { 11708 int map_idx = bnxt_cp_num_to_irq_num(bp, i); 11709 struct bnxt_irq *irq = &bp->irq_tbl[map_idx]; 11710 11711 if (IS_ENABLED(CONFIG_RFS_ACCEL) && 11712 rmap && bp->bnapi[i]->rx_ring) { 11713 rc = irq_cpu_rmap_add(rmap, irq->vector); 11714 if (rc) 11715 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n", 11716 j); 11717 j++; 11718 } 11719 11720 rc = request_irq(irq->vector, irq->handler, flags, irq->name, 11721 bp->bnapi[i]); 11722 if (rc) 11723 break; 11724 11725 netif_napi_set_irq_locked(&bp->bnapi[i]->napi, irq->vector); 11726 irq->requested = 1; 11727 11728 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) { 11729 int numa_node = dev_to_node(&bp->pdev->dev); 11730 u16 tag; 11731 11732 irq->have_cpumask = 1; 11733 irq->msix_nr = map_idx; 11734 irq->ring_nr = i; 11735 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 11736 irq->cpu_mask); 11737 rc = irq_update_affinity_hint(irq->vector, irq->cpu_mask); 11738 if (rc) { 11739 netdev_warn(bp->dev, 11740 "Update affinity hint failed, IRQ = %d\n", 11741 irq->vector); 11742 break; 11743 } 11744 11745 bnxt_register_irq_notifier(bp, irq); 11746 11747 /* Init ST table entry */ 11748 if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM, 11749 cpumask_first(irq->cpu_mask), 11750 &tag)) 11751 continue; 11752 11753 pcie_tph_set_st_entry(irq->bp->pdev, irq->msix_nr, tag); 11754 } 11755 } 11756 return rc; 11757} 11758 11759static void bnxt_del_napi(struct bnxt *bp) 11760{ 11761 int i; 11762 11763 if (!bp->bnapi) 11764 return; 11765 11766 for (i = 0; i < bp->rx_nr_rings; i++) 11767 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL); 11768 for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++) 11769 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL); 11770 11771 for (i = 0; i < bp->cp_nr_rings; i++) { 11772 struct bnxt_napi *bnapi = bp->bnapi[i]; 11773 11774 __netif_napi_del_locked(&bnapi->napi); 11775 } 11776 /* We called __netif_napi_del_locked(), we need 11777 * to respect an RCU grace period before freeing napi structures. 11778 */ 11779 synchronize_net(); 11780} 11781 11782static void bnxt_init_napi(struct bnxt *bp) 11783{ 11784 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll; 11785 unsigned int cp_nr_rings = bp->cp_nr_rings; 11786 struct bnxt_napi *bnapi; 11787 int i; 11788 11789 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 11790 poll_fn = bnxt_poll_p5; 11791 else if (BNXT_CHIP_TYPE_NITRO_A0(bp)) 11792 cp_nr_rings--; 11793 11794 set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 11795 11796 for (i = 0; i < cp_nr_rings; i++) { 11797 bnapi = bp->bnapi[i]; 11798 netif_napi_add_config_locked(bp->dev, &bnapi->napi, poll_fn, 11799 bnapi->index); 11800 } 11801 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 11802 bnapi = bp->bnapi[cp_nr_rings]; 11803 netif_napi_add_locked(bp->dev, &bnapi->napi, bnxt_poll_nitroa0); 11804 } 11805} 11806 11807static void bnxt_disable_napi(struct bnxt *bp) 11808{ 11809 int i; 11810 11811 if (!bp->bnapi || 11812 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state)) 11813 return; 11814 11815 for (i = 0; i < bp->cp_nr_rings; i++) { 11816 struct bnxt_napi *bnapi = bp->bnapi[i]; 11817 struct bnxt_cp_ring_info *cpr; 11818 11819 cpr = &bnapi->cp_ring; 11820 if (bnapi->tx_fault) 11821 cpr->sw_stats->tx.tx_resets++; 11822 if (bnapi->in_reset) 11823 cpr->sw_stats->rx.rx_resets++; 11824 napi_disable_locked(&bnapi->napi); 11825 } 11826} 11827 11828static void bnxt_enable_napi(struct bnxt *bp) 11829{ 11830 int i; 11831 11832 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state); 11833 for (i = 0; i < bp->cp_nr_rings; i++) { 11834 struct bnxt_napi *bnapi = bp->bnapi[i]; 11835 struct bnxt_cp_ring_info *cpr; 11836 11837 bnapi->tx_fault = 0; 11838 11839 cpr = &bnapi->cp_ring; 11840 bnapi->in_reset = false; 11841 11842 if (bnapi->rx_ring) { 11843 INIT_WORK(&cpr->dim.work, bnxt_dim_work); 11844 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 11845 } 11846 napi_enable_locked(&bnapi->napi); 11847 } 11848} 11849 11850void bnxt_tx_disable(struct bnxt *bp) 11851{ 11852 int i; 11853 struct bnxt_tx_ring_info *txr; 11854 11855 if (bp->tx_ring) { 11856 for (i = 0; i < bp->tx_nr_rings; i++) { 11857 txr = &bp->tx_ring[i]; 11858 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING); 11859 } 11860 } 11861 /* Make sure napi polls see @dev_state change */ 11862 synchronize_net(); 11863 /* Drop carrier first to prevent TX timeout */ 11864 netif_carrier_off(bp->dev); 11865 /* Stop all TX queues */ 11866 netif_tx_disable(bp->dev); 11867} 11868 11869void bnxt_tx_enable(struct bnxt *bp) 11870{ 11871 int i; 11872 struct bnxt_tx_ring_info *txr; 11873 11874 for (i = 0; i < bp->tx_nr_rings; i++) { 11875 txr = &bp->tx_ring[i]; 11876 WRITE_ONCE(txr->dev_state, 0); 11877 } 11878 /* Make sure napi polls see @dev_state change */ 11879 synchronize_net(); 11880 netif_tx_wake_all_queues(bp->dev); 11881 if (BNXT_LINK_IS_UP(bp)) 11882 netif_carrier_on(bp->dev); 11883} 11884 11885static char *bnxt_report_fec(struct bnxt_link_info *link_info) 11886{ 11887 u8 active_fec = link_info->active_fec_sig_mode & 11888 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; 11889 11890 switch (active_fec) { 11891 default: 11892 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: 11893 return "None"; 11894 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: 11895 return "Clause 74 BaseR"; 11896 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: 11897 return "Clause 91 RS(528,514)"; 11898 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: 11899 return "Clause 91 RS544_1XN"; 11900 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: 11901 return "Clause 91 RS(544,514)"; 11902 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: 11903 return "Clause 91 RS272_1XN"; 11904 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: 11905 return "Clause 91 RS(272,257)"; 11906 } 11907} 11908 11909void bnxt_report_link(struct bnxt *bp) 11910{ 11911 if (BNXT_LINK_IS_UP(bp)) { 11912 const char *signal = ""; 11913 const char *flow_ctrl; 11914 const char *duplex; 11915 u32 speed; 11916 u16 fec; 11917 11918 netif_carrier_on(bp->dev); 11919 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed); 11920 if (speed == SPEED_UNKNOWN) { 11921 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n"); 11922 return; 11923 } 11924 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL) 11925 duplex = "full"; 11926 else 11927 duplex = "half"; 11928 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH) 11929 flow_ctrl = "ON - receive & transmit"; 11930 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX) 11931 flow_ctrl = "ON - transmit"; 11932 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX) 11933 flow_ctrl = "ON - receive"; 11934 else 11935 flow_ctrl = "none"; 11936 if (bp->link_info.phy_qcfg_resp.option_flags & 11937 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { 11938 u8 sig_mode = bp->link_info.active_fec_sig_mode & 11939 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; 11940 switch (sig_mode) { 11941 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: 11942 signal = "(NRZ) "; 11943 break; 11944 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: 11945 signal = "(PAM4 56Gbps) "; 11946 break; 11947 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112: 11948 signal = "(PAM4 112Gbps) "; 11949 break; 11950 default: 11951 break; 11952 } 11953 } 11954 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", 11955 speed, signal, duplex, flow_ctrl); 11956 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) 11957 netdev_info(bp->dev, "EEE is %s\n", 11958 bp->eee.eee_active ? "active" : 11959 "not active"); 11960 fec = bp->link_info.fec_cfg; 11961 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) 11962 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n", 11963 (fec & BNXT_FEC_AUTONEG) ? "on" : "off", 11964 bnxt_report_fec(&bp->link_info)); 11965 } else { 11966 netif_carrier_off(bp->dev); 11967 netdev_err(bp->dev, "NIC Link is Down\n"); 11968 } 11969} 11970 11971static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) 11972{ 11973 if (!resp->supported_speeds_auto_mode && 11974 !resp->supported_speeds_force_mode && 11975 !resp->supported_pam4_speeds_auto_mode && 11976 !resp->supported_pam4_speeds_force_mode && 11977 !resp->supported_speeds2_auto_mode && 11978 !resp->supported_speeds2_force_mode) 11979 return true; 11980 return false; 11981} 11982 11983static int bnxt_hwrm_phy_qcaps(struct bnxt *bp) 11984{ 11985 struct bnxt_link_info *link_info = &bp->link_info; 11986 struct hwrm_port_phy_qcaps_output *resp; 11987 struct hwrm_port_phy_qcaps_input *req; 11988 int rc = 0; 11989 11990 if (bp->hwrm_spec_code < 0x10201) 11991 return 0; 11992 11993 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS); 11994 if (rc) 11995 return rc; 11996 11997 resp = hwrm_req_hold(bp, req); 11998 rc = hwrm_req_send(bp, req); 11999 if (rc) 12000 goto hwrm_phy_qcaps_exit; 12001 12002 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8); 12003 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) { 12004 struct ethtool_keee *eee = &bp->eee; 12005 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode); 12006 12007 _bnxt_fw_to_linkmode(eee->supported, fw_speeds); 12008 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) & 12009 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK; 12010 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) & 12011 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK; 12012 } 12013 12014 if (bp->hwrm_spec_code >= 0x10a01) { 12015 if (bnxt_phy_qcaps_no_speed(resp)) { 12016 link_info->phy_state = BNXT_PHY_STATE_DISABLED; 12017 netdev_warn(bp->dev, "Ethernet link disabled\n"); 12018 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { 12019 link_info->phy_state = BNXT_PHY_STATE_ENABLED; 12020 netdev_info(bp->dev, "Ethernet link enabled\n"); 12021 /* Phy re-enabled, reprobe the speeds */ 12022 link_info->support_auto_speeds = 0; 12023 link_info->support_pam4_auto_speeds = 0; 12024 link_info->support_auto_speeds2 = 0; 12025 } 12026 } 12027 if (resp->supported_speeds_auto_mode) 12028 link_info->support_auto_speeds = 12029 le16_to_cpu(resp->supported_speeds_auto_mode); 12030 if (resp->supported_pam4_speeds_auto_mode) 12031 link_info->support_pam4_auto_speeds = 12032 le16_to_cpu(resp->supported_pam4_speeds_auto_mode); 12033 if (resp->supported_speeds2_auto_mode) 12034 link_info->support_auto_speeds2 = 12035 le16_to_cpu(resp->supported_speeds2_auto_mode); 12036 12037 bp->port_count = resp->port_cnt; 12038 12039hwrm_phy_qcaps_exit: 12040 hwrm_req_drop(bp, req); 12041 return rc; 12042} 12043 12044static void bnxt_hwrm_mac_qcaps(struct bnxt *bp) 12045{ 12046 struct hwrm_port_mac_qcaps_output *resp; 12047 struct hwrm_port_mac_qcaps_input *req; 12048 int rc; 12049 12050 if (bp->hwrm_spec_code < 0x10a03) 12051 return; 12052 12053 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_QCAPS); 12054 if (rc) 12055 return; 12056 12057 resp = hwrm_req_hold(bp, req); 12058 rc = hwrm_req_send_silent(bp, req); 12059 if (!rc) 12060 bp->mac_flags = resp->flags; 12061 hwrm_req_drop(bp, req); 12062} 12063 12064static bool bnxt_support_dropped(u16 advertising, u16 supported) 12065{ 12066 u16 diff = advertising ^ supported; 12067 12068 return ((supported | diff) != supported); 12069} 12070 12071static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info) 12072{ 12073 struct bnxt *bp = container_of(link_info, struct bnxt, link_info); 12074 12075 /* Check if any advertised speeds are no longer supported. The caller 12076 * holds the link_lock mutex, so we can modify link_info settings. 12077 */ 12078 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 12079 if (bnxt_support_dropped(link_info->advertising, 12080 link_info->support_auto_speeds2)) { 12081 link_info->advertising = link_info->support_auto_speeds2; 12082 return true; 12083 } 12084 return false; 12085 } 12086 if (bnxt_support_dropped(link_info->advertising, 12087 link_info->support_auto_speeds)) { 12088 link_info->advertising = link_info->support_auto_speeds; 12089 return true; 12090 } 12091 if (bnxt_support_dropped(link_info->advertising_pam4, 12092 link_info->support_pam4_auto_speeds)) { 12093 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds; 12094 return true; 12095 } 12096 return false; 12097} 12098 12099int bnxt_update_link(struct bnxt *bp, bool chng_link_state) 12100{ 12101 struct bnxt_link_info *link_info = &bp->link_info; 12102 struct hwrm_port_phy_qcfg_output *resp; 12103 struct hwrm_port_phy_qcfg_input *req; 12104 u8 link_state = link_info->link_state; 12105 bool support_changed; 12106 int rc; 12107 12108 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG); 12109 if (rc) 12110 return rc; 12111 12112 resp = hwrm_req_hold(bp, req); 12113 rc = hwrm_req_send(bp, req); 12114 if (rc) { 12115 hwrm_req_drop(bp, req); 12116 if (BNXT_VF(bp) && rc == -ENODEV) { 12117 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n"); 12118 rc = 0; 12119 } 12120 return rc; 12121 } 12122 12123 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); 12124 link_info->phy_link_status = resp->link; 12125 link_info->duplex = resp->duplex_cfg; 12126 if (bp->hwrm_spec_code >= 0x10800) 12127 link_info->duplex = resp->duplex_state; 12128 link_info->pause = resp->pause; 12129 link_info->auto_mode = resp->auto_mode; 12130 link_info->auto_pause_setting = resp->auto_pause; 12131 link_info->lp_pause = resp->link_partner_adv_pause; 12132 link_info->force_pause_setting = resp->force_pause; 12133 link_info->duplex_setting = resp->duplex_cfg; 12134 if (link_info->phy_link_status == BNXT_LINK_LINK) { 12135 link_info->link_speed = le16_to_cpu(resp->link_speed); 12136 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) 12137 link_info->active_lanes = resp->active_lanes; 12138 } else { 12139 link_info->link_speed = 0; 12140 link_info->active_lanes = 0; 12141 } 12142 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed); 12143 link_info->force_pam4_link_speed = 12144 le16_to_cpu(resp->force_pam4_link_speed); 12145 link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2); 12146 link_info->support_speeds = le16_to_cpu(resp->support_speeds); 12147 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds); 12148 link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2); 12149 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask); 12150 link_info->auto_pam4_link_speeds = 12151 le16_to_cpu(resp->auto_pam4_link_speed_mask); 12152 link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2); 12153 link_info->lp_auto_link_speeds = 12154 le16_to_cpu(resp->link_partner_adv_speeds); 12155 link_info->lp_auto_pam4_link_speeds = 12156 resp->link_partner_pam4_adv_speeds; 12157 link_info->preemphasis = le32_to_cpu(resp->preemphasis); 12158 link_info->phy_ver[0] = resp->phy_maj; 12159 link_info->phy_ver[1] = resp->phy_min; 12160 link_info->phy_ver[2] = resp->phy_bld; 12161 link_info->media_type = resp->media_type; 12162 link_info->phy_type = resp->phy_type; 12163 link_info->transceiver = resp->xcvr_pkg_type; 12164 link_info->phy_addr = resp->eee_config_phy_addr & 12165 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; 12166 link_info->module_status = resp->module_status; 12167 12168 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) { 12169 struct ethtool_keee *eee = &bp->eee; 12170 u16 fw_speeds; 12171 12172 eee->eee_active = 0; 12173 if (resp->eee_config_phy_addr & 12174 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) { 12175 eee->eee_active = 1; 12176 fw_speeds = le16_to_cpu( 12177 resp->link_partner_adv_eee_link_speed_mask); 12178 _bnxt_fw_to_linkmode(eee->lp_advertised, fw_speeds); 12179 } 12180 12181 /* Pull initial EEE config */ 12182 if (!chng_link_state) { 12183 if (resp->eee_config_phy_addr & 12184 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED) 12185 eee->eee_enabled = 1; 12186 12187 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask); 12188 _bnxt_fw_to_linkmode(eee->advertised, fw_speeds); 12189 12190 if (resp->eee_config_phy_addr & 12191 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) { 12192 __le32 tmr; 12193 12194 eee->tx_lpi_enabled = 1; 12195 tmr = resp->xcvr_identifier_type_tx_lpi_timer; 12196 eee->tx_lpi_timer = le32_to_cpu(tmr) & 12197 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK; 12198 } 12199 } 12200 } 12201 12202 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED; 12203 if (bp->hwrm_spec_code >= 0x10504) { 12204 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); 12205 link_info->active_fec_sig_mode = resp->active_fec_signal_mode; 12206 } 12207 /* TODO: need to add more logic to report VF link */ 12208 if (chng_link_state) { 12209 if (link_info->phy_link_status == BNXT_LINK_LINK) 12210 link_info->link_state = BNXT_LINK_STATE_UP; 12211 else 12212 link_info->link_state = BNXT_LINK_STATE_DOWN; 12213 if (link_state != link_info->link_state) 12214 bnxt_report_link(bp); 12215 } else { 12216 /* always link down if not require to update link state */ 12217 link_info->link_state = BNXT_LINK_STATE_DOWN; 12218 } 12219 hwrm_req_drop(bp, req); 12220 12221 if (!BNXT_PHY_CFG_ABLE(bp)) 12222 return 0; 12223 12224 support_changed = bnxt_support_speed_dropped(link_info); 12225 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED)) 12226 bnxt_hwrm_set_link_setting(bp, true, false); 12227 return 0; 12228} 12229 12230static void bnxt_get_port_module_status(struct bnxt *bp) 12231{ 12232 struct bnxt_link_info *link_info = &bp->link_info; 12233 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; 12234 u8 module_status; 12235 12236 if (bnxt_update_link(bp, true)) 12237 return; 12238 12239 module_status = link_info->module_status; 12240 switch (module_status) { 12241 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: 12242 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: 12243 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: 12244 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n", 12245 bp->pf.port_id); 12246 if (bp->hwrm_spec_code >= 0x10201) { 12247 netdev_warn(bp->dev, "Module part number %s\n", 12248 resp->phy_vendor_partnumber); 12249 } 12250 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) 12251 netdev_warn(bp->dev, "TX is disabled\n"); 12252 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) 12253 netdev_warn(bp->dev, "SFP+ module is shutdown\n"); 12254 } 12255} 12256 12257static void 12258bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 12259{ 12260 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) { 12261 if (bp->hwrm_spec_code >= 0x10201) 12262 req->auto_pause = 12263 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; 12264 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 12265 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; 12266 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 12267 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; 12268 req->enables |= 12269 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 12270 } else { 12271 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX) 12272 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; 12273 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX) 12274 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; 12275 req->enables |= 12276 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); 12277 if (bp->hwrm_spec_code >= 0x10201) { 12278 req->auto_pause = req->force_pause; 12279 req->enables |= cpu_to_le32( 12280 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); 12281 } 12282 } 12283} 12284 12285static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req) 12286{ 12287 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) { 12288 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; 12289 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 12290 req->enables |= 12291 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK); 12292 req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising); 12293 } else if (bp->link_info.advertising) { 12294 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK); 12295 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising); 12296 } 12297 if (bp->link_info.advertising_pam4) { 12298 req->enables |= 12299 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK); 12300 req->auto_link_pam4_speed_mask = 12301 cpu_to_le16(bp->link_info.advertising_pam4); 12302 } 12303 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); 12304 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG); 12305 } else { 12306 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); 12307 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) { 12308 req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed); 12309 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2); 12310 netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n", 12311 (u32)bp->link_info.req_link_speed); 12312 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) { 12313 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 12314 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED); 12315 } else { 12316 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed); 12317 } 12318 } 12319 12320 /* tell chimp that the setting takes effect immediately */ 12321 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); 12322} 12323 12324int bnxt_hwrm_set_pause(struct bnxt *bp) 12325{ 12326 struct hwrm_port_phy_cfg_input *req; 12327 int rc; 12328 12329 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 12330 if (rc) 12331 return rc; 12332 12333 bnxt_hwrm_set_pause_common(bp, req); 12334 12335 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) || 12336 bp->link_info.force_link_chng) 12337 bnxt_hwrm_set_link_common(bp, req); 12338 12339 rc = hwrm_req_send(bp, req); 12340 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) { 12341 /* since changing of pause setting doesn't trigger any link 12342 * change event, the driver needs to update the current pause 12343 * result upon successfully return of the phy_cfg command 12344 */ 12345 bp->link_info.pause = 12346 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl; 12347 bp->link_info.auto_pause_setting = 0; 12348 if (!bp->link_info.force_link_chng) 12349 bnxt_report_link(bp); 12350 } 12351 bp->link_info.force_link_chng = false; 12352 return rc; 12353} 12354 12355static void bnxt_hwrm_set_eee(struct bnxt *bp, 12356 struct hwrm_port_phy_cfg_input *req) 12357{ 12358 struct ethtool_keee *eee = &bp->eee; 12359 12360 if (eee->eee_enabled) { 12361 u16 eee_speeds; 12362 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE; 12363 12364 if (eee->tx_lpi_enabled) 12365 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE; 12366 else 12367 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE; 12368 12369 req->flags |= cpu_to_le32(flags); 12370 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); 12371 req->eee_link_speed_mask = cpu_to_le16(eee_speeds); 12372 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer); 12373 } else { 12374 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE); 12375 } 12376} 12377 12378int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee) 12379{ 12380 struct hwrm_port_phy_cfg_input *req; 12381 int rc; 12382 12383 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 12384 if (rc) 12385 return rc; 12386 12387 if (set_pause) 12388 bnxt_hwrm_set_pause_common(bp, req); 12389 12390 bnxt_hwrm_set_link_common(bp, req); 12391 12392 if (set_eee) 12393 bnxt_hwrm_set_eee(bp, req); 12394 return hwrm_req_send(bp, req); 12395} 12396 12397static int bnxt_hwrm_shutdown_link(struct bnxt *bp) 12398{ 12399 struct hwrm_port_phy_cfg_input *req; 12400 int rc; 12401 12402 if (!BNXT_SINGLE_PF(bp)) 12403 return 0; 12404 12405 if (pci_num_vf(bp->pdev) && 12406 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN)) 12407 return 0; 12408 12409 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG); 12410 if (rc) 12411 return rc; 12412 12413 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); 12414 rc = hwrm_req_send(bp, req); 12415 if (!rc) { 12416 mutex_lock(&bp->link_lock); 12417 /* Device is not obliged link down in certain scenarios, even 12418 * when forced. Setting the state unknown is consistent with 12419 * driver startup and will force link state to be reported 12420 * during subsequent open based on PORT_PHY_QCFG. 12421 */ 12422 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN; 12423 mutex_unlock(&bp->link_lock); 12424 } 12425 return rc; 12426} 12427 12428static int bnxt_fw_reset_via_optee(struct bnxt *bp) 12429{ 12430#ifdef CONFIG_TEE_BNXT_FW 12431 int rc = tee_bnxt_fw_load(); 12432 12433 if (rc) 12434 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc); 12435 12436 return rc; 12437#else 12438 netdev_err(bp->dev, "OP-TEE not supported\n"); 12439 return -ENODEV; 12440#endif 12441} 12442 12443static int bnxt_try_recover_fw(struct bnxt *bp) 12444{ 12445 if (bp->fw_health && bp->fw_health->status_reliable) { 12446 int retry = 0, rc; 12447 u32 sts; 12448 12449 do { 12450 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 12451 rc = bnxt_hwrm_poll(bp); 12452 if (!BNXT_FW_IS_BOOTING(sts) && 12453 !BNXT_FW_IS_RECOVERING(sts)) 12454 break; 12455 retry++; 12456 } while (rc == -EBUSY && retry < BNXT_FW_RETRY); 12457 12458 if (!BNXT_FW_IS_HEALTHY(sts)) { 12459 netdev_err(bp->dev, 12460 "Firmware not responding, status: 0x%x\n", 12461 sts); 12462 rc = -ENODEV; 12463 } 12464 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) { 12465 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n"); 12466 return bnxt_fw_reset_via_optee(bp); 12467 } 12468 return rc; 12469 } 12470 12471 return -ENODEV; 12472} 12473 12474void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset) 12475{ 12476 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 12477 12478 if (!BNXT_NEW_RM(bp)) 12479 return; /* no resource reservations required */ 12480 12481 hw_resc->resv_cp_rings = 0; 12482 hw_resc->resv_stat_ctxs = 0; 12483 hw_resc->resv_irqs = 0; 12484 hw_resc->resv_tx_rings = 0; 12485 hw_resc->resv_rx_rings = 0; 12486 hw_resc->resv_hw_ring_grps = 0; 12487 hw_resc->resv_vnics = 0; 12488 hw_resc->resv_rsscos_ctxs = 0; 12489 if (!fw_reset) { 12490 bp->tx_nr_rings = 0; 12491 bp->rx_nr_rings = 0; 12492 } 12493} 12494 12495int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) 12496{ 12497 int rc; 12498 12499 if (!BNXT_NEW_RM(bp)) 12500 return 0; /* no resource reservations required */ 12501 12502 rc = bnxt_hwrm_func_resc_qcaps(bp, true); 12503 if (rc) 12504 netdev_err(bp->dev, "resc_qcaps failed\n"); 12505 12506 bnxt_clear_reservations(bp, fw_reset); 12507 12508 return rc; 12509} 12510 12511static int bnxt_hwrm_if_change(struct bnxt *bp, bool up) 12512{ 12513 struct hwrm_func_drv_if_change_output *resp; 12514 struct hwrm_func_drv_if_change_input *req; 12515 bool resc_reinit = false; 12516 bool caps_change = false; 12517 int rc, retry = 0; 12518 bool fw_reset; 12519 u32 flags = 0; 12520 12521 fw_reset = (bp->fw_reset_state == BNXT_FW_RESET_STATE_ABORT); 12522 bp->fw_reset_state = 0; 12523 12524 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE)) 12525 return 0; 12526 12527 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE); 12528 if (rc) 12529 return rc; 12530 12531 if (up) 12532 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP); 12533 resp = hwrm_req_hold(bp, req); 12534 12535 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT); 12536 while (retry < BNXT_FW_IF_RETRY) { 12537 rc = hwrm_req_send(bp, req); 12538 if (rc != -EAGAIN) 12539 break; 12540 12541 msleep(50); 12542 retry++; 12543 } 12544 12545 if (rc == -EAGAIN) { 12546 hwrm_req_drop(bp, req); 12547 return rc; 12548 } else if (!rc) { 12549 flags = le32_to_cpu(resp->flags); 12550 } else if (up) { 12551 rc = bnxt_try_recover_fw(bp); 12552 fw_reset = true; 12553 } 12554 hwrm_req_drop(bp, req); 12555 if (rc) 12556 return rc; 12557 12558 if (!up) { 12559 bnxt_inv_fw_health_reg(bp); 12560 return 0; 12561 } 12562 12563 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE) 12564 resc_reinit = true; 12565 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE || 12566 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) 12567 fw_reset = true; 12568 else 12569 bnxt_remap_fw_health_regs(bp); 12570 12571 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) { 12572 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n"); 12573 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 12574 return -ENODEV; 12575 } 12576 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_CAPS_CHANGE) 12577 caps_change = true; 12578 12579 if (resc_reinit || fw_reset || caps_change) { 12580 if (fw_reset || caps_change) { 12581 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 12582 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 12583 bnxt_ulp_irq_stop(bp); 12584 bnxt_free_ctx_mem(bp, false); 12585 bnxt_dcb_free(bp); 12586 rc = bnxt_fw_init_one(bp); 12587 if (rc) { 12588 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 12589 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 12590 return rc; 12591 } 12592 /* IRQ will be initialized later in bnxt_request_irq()*/ 12593 bnxt_clear_int_mode(bp); 12594 } 12595 rc = bnxt_cancel_reservations(bp, fw_reset); 12596 } 12597 return rc; 12598} 12599 12600static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp) 12601{ 12602 struct hwrm_port_led_qcaps_output *resp; 12603 struct hwrm_port_led_qcaps_input *req; 12604 struct bnxt_pf_info *pf = &bp->pf; 12605 int rc; 12606 12607 bp->num_leds = 0; 12608 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601) 12609 return 0; 12610 12611 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS); 12612 if (rc) 12613 return rc; 12614 12615 req->port_id = cpu_to_le16(pf->port_id); 12616 resp = hwrm_req_hold(bp, req); 12617 rc = hwrm_req_send(bp, req); 12618 if (rc) { 12619 hwrm_req_drop(bp, req); 12620 return rc; 12621 } 12622 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) { 12623 int i; 12624 12625 bp->num_leds = resp->num_leds; 12626 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) * 12627 bp->num_leds); 12628 for (i = 0; i < bp->num_leds; i++) { 12629 struct bnxt_led_info *led = &bp->leds[i]; 12630 __le16 caps = led->led_state_caps; 12631 12632 if (!led->led_group_id || 12633 !BNXT_LED_ALT_BLINK_CAP(caps)) { 12634 bp->num_leds = 0; 12635 break; 12636 } 12637 } 12638 } 12639 hwrm_req_drop(bp, req); 12640 return 0; 12641} 12642 12643int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp) 12644{ 12645 struct hwrm_wol_filter_alloc_output *resp; 12646 struct hwrm_wol_filter_alloc_input *req; 12647 int rc; 12648 12649 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC); 12650 if (rc) 12651 return rc; 12652 12653 req->port_id = cpu_to_le16(bp->pf.port_id); 12654 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT; 12655 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS); 12656 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN); 12657 12658 resp = hwrm_req_hold(bp, req); 12659 rc = hwrm_req_send(bp, req); 12660 if (!rc) 12661 bp->wol_filter_id = resp->wol_filter_id; 12662 hwrm_req_drop(bp, req); 12663 return rc; 12664} 12665 12666int bnxt_hwrm_free_wol_fltr(struct bnxt *bp) 12667{ 12668 struct hwrm_wol_filter_free_input *req; 12669 int rc; 12670 12671 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE); 12672 if (rc) 12673 return rc; 12674 12675 req->port_id = cpu_to_le16(bp->pf.port_id); 12676 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID); 12677 req->wol_filter_id = bp->wol_filter_id; 12678 12679 return hwrm_req_send(bp, req); 12680} 12681 12682static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle) 12683{ 12684 struct hwrm_wol_filter_qcfg_output *resp; 12685 struct hwrm_wol_filter_qcfg_input *req; 12686 u16 next_handle = 0; 12687 int rc; 12688 12689 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG); 12690 if (rc) 12691 return rc; 12692 12693 req->port_id = cpu_to_le16(bp->pf.port_id); 12694 req->handle = cpu_to_le16(handle); 12695 resp = hwrm_req_hold(bp, req); 12696 rc = hwrm_req_send(bp, req); 12697 if (!rc) { 12698 next_handle = le16_to_cpu(resp->next_handle); 12699 if (next_handle != 0) { 12700 if (resp->wol_type == 12701 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) { 12702 bp->wol = 1; 12703 bp->wol_filter_id = resp->wol_filter_id; 12704 } 12705 } 12706 } 12707 hwrm_req_drop(bp, req); 12708 return next_handle; 12709} 12710 12711static void bnxt_get_wol_settings(struct bnxt *bp) 12712{ 12713 u16 handle = 0; 12714 12715 bp->wol = 0; 12716 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP)) 12717 return; 12718 12719 do { 12720 handle = bnxt_hwrm_get_wol_fltrs(bp, handle); 12721 } while (handle && handle != 0xffff); 12722} 12723 12724static bool bnxt_eee_config_ok(struct bnxt *bp) 12725{ 12726 struct ethtool_keee *eee = &bp->eee; 12727 struct bnxt_link_info *link_info = &bp->link_info; 12728 12729 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP)) 12730 return true; 12731 12732 if (eee->eee_enabled) { 12733 __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); 12734 __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp); 12735 12736 _bnxt_fw_to_linkmode(advertising, link_info->advertising); 12737 12738 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 12739 eee->eee_enabled = 0; 12740 return false; 12741 } 12742 if (linkmode_andnot(tmp, eee->advertised, advertising)) { 12743 linkmode_and(eee->advertised, advertising, 12744 eee->supported); 12745 return false; 12746 } 12747 } 12748 return true; 12749} 12750 12751static int bnxt_update_phy_setting(struct bnxt *bp) 12752{ 12753 int rc; 12754 bool update_link = false; 12755 bool update_pause = false; 12756 bool update_eee = false; 12757 struct bnxt_link_info *link_info = &bp->link_info; 12758 12759 rc = bnxt_update_link(bp, true); 12760 if (rc) { 12761 netdev_err(bp->dev, "failed to update link (rc: %x)\n", 12762 rc); 12763 return rc; 12764 } 12765 if (!BNXT_SINGLE_PF(bp)) 12766 return 0; 12767 12768 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 12769 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) != 12770 link_info->req_flow_ctrl) 12771 update_pause = true; 12772 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) && 12773 link_info->force_pause_setting != link_info->req_flow_ctrl) 12774 update_pause = true; 12775 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) { 12776 if (BNXT_AUTO_MODE(link_info->auto_mode)) 12777 update_link = true; 12778 if (bnxt_force_speed_updated(link_info)) 12779 update_link = true; 12780 if (link_info->req_duplex != link_info->duplex_setting) 12781 update_link = true; 12782 } else { 12783 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE) 12784 update_link = true; 12785 if (bnxt_auto_speed_updated(link_info)) 12786 update_link = true; 12787 } 12788 12789 /* The last close may have shutdown the link, so need to call 12790 * PHY_CFG to bring it back up. 12791 */ 12792 if (!BNXT_LINK_IS_UP(bp)) 12793 update_link = true; 12794 12795 if (!bnxt_eee_config_ok(bp)) 12796 update_eee = true; 12797 12798 if (update_link) 12799 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee); 12800 else if (update_pause) 12801 rc = bnxt_hwrm_set_pause(bp); 12802 if (rc) { 12803 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n", 12804 rc); 12805 return rc; 12806 } 12807 12808 return rc; 12809} 12810 12811static int bnxt_init_dflt_ring_mode(struct bnxt *bp); 12812 12813static int bnxt_reinit_after_abort(struct bnxt *bp) 12814{ 12815 int rc; 12816 12817 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 12818 return -EBUSY; 12819 12820 if (bp->dev->reg_state == NETREG_UNREGISTERED) 12821 return -ENODEV; 12822 12823 rc = bnxt_fw_init_one(bp); 12824 if (!rc) { 12825 bnxt_clear_int_mode(bp); 12826 rc = bnxt_init_int_mode(bp); 12827 if (!rc) { 12828 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state); 12829 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state); 12830 } 12831 } 12832 return rc; 12833} 12834 12835static void bnxt_cfg_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr) 12836{ 12837 struct bnxt_ntuple_filter *ntp_fltr; 12838 struct bnxt_l2_filter *l2_fltr; 12839 12840 if (list_empty(&fltr->list)) 12841 return; 12842 12843 if (fltr->type == BNXT_FLTR_TYPE_NTUPLE) { 12844 ntp_fltr = container_of(fltr, struct bnxt_ntuple_filter, base); 12845 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0]; 12846 atomic_inc(&l2_fltr->refcnt); 12847 ntp_fltr->l2_fltr = l2_fltr; 12848 if (bnxt_hwrm_cfa_ntuple_filter_alloc(bp, ntp_fltr)) { 12849 bnxt_del_ntp_filter(bp, ntp_fltr); 12850 netdev_err(bp->dev, "restoring previously configured ntuple filter id %d failed\n", 12851 fltr->sw_id); 12852 } 12853 } else if (fltr->type == BNXT_FLTR_TYPE_L2) { 12854 l2_fltr = container_of(fltr, struct bnxt_l2_filter, base); 12855 if (bnxt_hwrm_l2_filter_alloc(bp, l2_fltr)) { 12856 bnxt_del_l2_filter(bp, l2_fltr); 12857 netdev_err(bp->dev, "restoring previously configured l2 filter id %d failed\n", 12858 fltr->sw_id); 12859 } 12860 } 12861} 12862 12863static void bnxt_cfg_usr_fltrs(struct bnxt *bp) 12864{ 12865 struct bnxt_filter_base *usr_fltr, *tmp; 12866 12867 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) 12868 bnxt_cfg_one_usr_fltr(bp, usr_fltr); 12869} 12870 12871static int bnxt_set_xps_mapping(struct bnxt *bp) 12872{ 12873 int numa_node = dev_to_node(&bp->pdev->dev); 12874 unsigned int q_idx, map_idx, cpu, i; 12875 const struct cpumask *cpu_mask_ptr; 12876 int nr_cpus = num_online_cpus(); 12877 cpumask_t *q_map; 12878 int rc = 0; 12879 12880 q_map = kcalloc(bp->tx_nr_rings_per_tc, sizeof(*q_map), GFP_KERNEL); 12881 if (!q_map) 12882 return -ENOMEM; 12883 12884 /* Create CPU mask for all TX queues across MQPRIO traffic classes. 12885 * Each TC has the same number of TX queues. The nth TX queue for each 12886 * TC will have the same CPU mask. 12887 */ 12888 for (i = 0; i < nr_cpus; i++) { 12889 map_idx = i % bp->tx_nr_rings_per_tc; 12890 cpu = cpumask_local_spread(i, numa_node); 12891 cpu_mask_ptr = get_cpu_mask(cpu); 12892 cpumask_or(&q_map[map_idx], &q_map[map_idx], cpu_mask_ptr); 12893 } 12894 12895 /* Register CPU mask for each TX queue except the ones marked for XDP */ 12896 for (q_idx = 0; q_idx < bp->dev->real_num_tx_queues; q_idx++) { 12897 map_idx = q_idx % bp->tx_nr_rings_per_tc; 12898 rc = netif_set_xps_queue(bp->dev, &q_map[map_idx], q_idx); 12899 if (rc) { 12900 netdev_warn(bp->dev, "Error setting XPS for q:%d\n", 12901 q_idx); 12902 break; 12903 } 12904 } 12905 12906 kfree(q_map); 12907 12908 return rc; 12909} 12910 12911static int bnxt_tx_nr_rings(struct bnxt *bp) 12912{ 12913 return bp->num_tc ? bp->tx_nr_rings_per_tc * bp->num_tc : 12914 bp->tx_nr_rings_per_tc; 12915} 12916 12917static int bnxt_tx_nr_rings_per_tc(struct bnxt *bp) 12918{ 12919 return bp->num_tc ? bp->tx_nr_rings / bp->num_tc : bp->tx_nr_rings; 12920} 12921 12922static void bnxt_set_xdp_tx_rings(struct bnxt *bp) 12923{ 12924 bp->tx_nr_rings_xdp = bp->tx_nr_rings_per_tc; 12925 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 12926} 12927 12928static void bnxt_adj_tx_rings(struct bnxt *bp) 12929{ 12930 /* Make adjustments if reserved TX rings are less than requested */ 12931 bp->tx_nr_rings -= bp->tx_nr_rings_xdp; 12932 bp->tx_nr_rings_per_tc = bnxt_tx_nr_rings_per_tc(bp); 12933 if (bp->tx_nr_rings_xdp) 12934 bnxt_set_xdp_tx_rings(bp); 12935} 12936 12937static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 12938{ 12939 int rc = 0; 12940 12941 netif_carrier_off(bp->dev); 12942 if (irq_re_init) { 12943 /* Reserve rings now if none were reserved at driver probe. */ 12944 rc = bnxt_init_dflt_ring_mode(bp); 12945 if (rc) { 12946 netdev_err(bp->dev, "Failed to reserve default rings at open\n"); 12947 return rc; 12948 } 12949 } 12950 rc = bnxt_reserve_rings(bp, irq_re_init); 12951 if (rc) 12952 return rc; 12953 12954 bnxt_adj_tx_rings(bp); 12955 rc = bnxt_alloc_mem(bp, irq_re_init); 12956 if (rc) { 12957 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 12958 goto open_err_free_mem; 12959 } 12960 12961 if (irq_re_init) { 12962 bnxt_init_napi(bp); 12963 rc = bnxt_request_irq(bp); 12964 if (rc) { 12965 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc); 12966 goto open_err_irq; 12967 } 12968 } 12969 12970 rc = bnxt_init_nic(bp, irq_re_init); 12971 if (rc) { 12972 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 12973 goto open_err_irq; 12974 } 12975 12976 bnxt_enable_napi(bp); 12977 bnxt_debug_dev_init(bp); 12978 12979 if (link_re_init) { 12980 mutex_lock(&bp->link_lock); 12981 rc = bnxt_update_phy_setting(bp); 12982 mutex_unlock(&bp->link_lock); 12983 if (rc) { 12984 netdev_warn(bp->dev, "failed to update phy settings\n"); 12985 if (BNXT_SINGLE_PF(bp)) { 12986 bp->link_info.phy_retry = true; 12987 bp->link_info.phy_retry_expires = 12988 jiffies + 5 * HZ; 12989 } 12990 } 12991 } 12992 12993 if (irq_re_init) { 12994 udp_tunnel_nic_reset_ntf(bp->dev); 12995 rc = bnxt_set_xps_mapping(bp); 12996 if (rc) 12997 netdev_warn(bp->dev, "failed to set xps mapping\n"); 12998 } 12999 13000 if (bp->tx_nr_rings_xdp < num_possible_cpus()) { 13001 if (!static_key_enabled(&bnxt_xdp_locking_key)) 13002 static_branch_enable(&bnxt_xdp_locking_key); 13003 } else if (static_key_enabled(&bnxt_xdp_locking_key)) { 13004 static_branch_disable(&bnxt_xdp_locking_key); 13005 } 13006 set_bit(BNXT_STATE_OPEN, &bp->state); 13007 bnxt_enable_int(bp); 13008 /* Enable TX queues */ 13009 bnxt_tx_enable(bp); 13010 mod_timer(&bp->timer, jiffies + bp->current_interval); 13011 /* Poll link status and check for SFP+ module status */ 13012 mutex_lock(&bp->link_lock); 13013 bnxt_get_port_module_status(bp); 13014 mutex_unlock(&bp->link_lock); 13015 13016 /* VF-reps may need to be re-opened after the PF is re-opened */ 13017 if (BNXT_PF(bp)) 13018 bnxt_vf_reps_open(bp); 13019 bnxt_ptp_init_rtc(bp, true); 13020 bnxt_ptp_cfg_tstamp_filters(bp); 13021 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp)) 13022 bnxt_hwrm_realloc_rss_ctx_vnic(bp); 13023 bnxt_cfg_usr_fltrs(bp); 13024 return 0; 13025 13026open_err_irq: 13027 bnxt_del_napi(bp); 13028 13029open_err_free_mem: 13030 bnxt_free_skbs(bp); 13031 bnxt_free_irq(bp); 13032 bnxt_free_mem(bp, true); 13033 return rc; 13034} 13035 13036int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 13037{ 13038 int rc = 0; 13039 13040 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) 13041 rc = -EIO; 13042 if (!rc) 13043 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init); 13044 if (rc) { 13045 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc); 13046 netif_close(bp->dev); 13047 } 13048 return rc; 13049} 13050 13051/* netdev instance lock held, open the NIC half way by allocating all 13052 * resources, but NAPI, IRQ, and TX are not enabled. This is mainly used 13053 * for offline self tests. 13054 */ 13055int bnxt_half_open_nic(struct bnxt *bp) 13056{ 13057 int rc = 0; 13058 13059 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 13060 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n"); 13061 rc = -ENODEV; 13062 goto half_open_err; 13063 } 13064 13065 rc = bnxt_alloc_mem(bp, true); 13066 if (rc) { 13067 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc); 13068 goto half_open_err; 13069 } 13070 bnxt_init_napi(bp); 13071 set_bit(BNXT_STATE_HALF_OPEN, &bp->state); 13072 rc = bnxt_init_nic(bp, true); 13073 if (rc) { 13074 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 13075 bnxt_del_napi(bp); 13076 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc); 13077 goto half_open_err; 13078 } 13079 return 0; 13080 13081half_open_err: 13082 bnxt_free_skbs(bp); 13083 bnxt_free_mem(bp, true); 13084 netif_close(bp->dev); 13085 return rc; 13086} 13087 13088/* netdev instance lock held, this call can only be made after a previous 13089 * successful call to bnxt_half_open_nic(). 13090 */ 13091void bnxt_half_close_nic(struct bnxt *bp) 13092{ 13093 bnxt_hwrm_resource_free(bp, false, true); 13094 bnxt_del_napi(bp); 13095 bnxt_free_skbs(bp); 13096 bnxt_free_mem(bp, true); 13097 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state); 13098} 13099 13100void bnxt_reenable_sriov(struct bnxt *bp) 13101{ 13102 if (BNXT_PF(bp)) { 13103 struct bnxt_pf_info *pf = &bp->pf; 13104 int n = pf->active_vfs; 13105 13106 if (n) 13107 bnxt_cfg_hw_sriov(bp, &n, true); 13108 } 13109} 13110 13111static int bnxt_open(struct net_device *dev) 13112{ 13113 struct bnxt *bp = netdev_priv(dev); 13114 int rc; 13115 13116 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 13117 rc = bnxt_reinit_after_abort(bp); 13118 if (rc) { 13119 if (rc == -EBUSY) 13120 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n"); 13121 else 13122 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n"); 13123 return -ENODEV; 13124 } 13125 } 13126 13127 rc = bnxt_hwrm_if_change(bp, true); 13128 if (rc) 13129 return rc; 13130 13131 rc = __bnxt_open_nic(bp, true, true); 13132 if (rc) { 13133 bnxt_hwrm_if_change(bp, false); 13134 } else { 13135 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { 13136 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 13137 bnxt_queue_sp_work(bp, 13138 BNXT_RESTART_ULP_SP_EVENT); 13139 } 13140 } 13141 13142 return rc; 13143} 13144 13145static bool bnxt_drv_busy(struct bnxt *bp) 13146{ 13147 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) || 13148 test_bit(BNXT_STATE_READ_STATS, &bp->state)); 13149} 13150 13151static void bnxt_get_ring_stats(struct bnxt *bp, 13152 struct rtnl_link_stats64 *stats); 13153 13154static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, 13155 bool link_re_init) 13156{ 13157 /* Close the VF-reps before closing PF */ 13158 if (BNXT_PF(bp)) 13159 bnxt_vf_reps_close(bp); 13160 13161 /* Change device state to avoid TX queue wake up's */ 13162 bnxt_tx_disable(bp); 13163 13164 clear_bit(BNXT_STATE_OPEN, &bp->state); 13165 smp_mb__after_atomic(); 13166 while (bnxt_drv_busy(bp)) 13167 msleep(20); 13168 13169 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp)) 13170 bnxt_clear_rss_ctxs(bp); 13171 /* Flush rings and disable interrupts */ 13172 bnxt_shutdown_nic(bp, irq_re_init); 13173 13174 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */ 13175 13176 bnxt_debug_dev_exit(bp); 13177 bnxt_disable_napi(bp); 13178 timer_delete_sync(&bp->timer); 13179 bnxt_free_skbs(bp); 13180 13181 /* Save ring stats before shutdown */ 13182 if (bp->bnapi && irq_re_init) { 13183 bnxt_get_ring_stats(bp, &bp->net_stats_prev); 13184 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev); 13185 } 13186 if (irq_re_init) { 13187 bnxt_free_irq(bp); 13188 bnxt_del_napi(bp); 13189 } 13190 bnxt_free_mem(bp, irq_re_init); 13191} 13192 13193void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) 13194{ 13195 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 13196 /* If we get here, it means firmware reset is in progress 13197 * while we are trying to close. We can safely proceed with 13198 * the close because we are holding netdev instance lock. 13199 * Some firmware messages may fail as we proceed to close. 13200 * We set the ABORT_ERR flag here so that the FW reset thread 13201 * will later abort when it gets the netdev instance lock 13202 * and sees the flag. 13203 */ 13204 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n"); 13205 set_bit(BNXT_STATE_ABORT_ERR, &bp->state); 13206 } 13207 13208#ifdef CONFIG_BNXT_SRIOV 13209 if (bp->sriov_cfg) { 13210 int rc; 13211 13212 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait, 13213 !bp->sriov_cfg, 13214 BNXT_SRIOV_CFG_WAIT_TMO); 13215 if (!rc) 13216 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n"); 13217 else if (rc < 0) 13218 netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n"); 13219 } 13220#endif 13221 __bnxt_close_nic(bp, irq_re_init, link_re_init); 13222} 13223 13224static int bnxt_close(struct net_device *dev) 13225{ 13226 struct bnxt *bp = netdev_priv(dev); 13227 13228 bnxt_close_nic(bp, true, true); 13229 bnxt_hwrm_shutdown_link(bp); 13230 bnxt_hwrm_if_change(bp, false); 13231 return 0; 13232} 13233 13234static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg, 13235 u16 *val) 13236{ 13237 struct hwrm_port_phy_mdio_read_output *resp; 13238 struct hwrm_port_phy_mdio_read_input *req; 13239 int rc; 13240 13241 if (bp->hwrm_spec_code < 0x10a00) 13242 return -EOPNOTSUPP; 13243 13244 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ); 13245 if (rc) 13246 return rc; 13247 13248 req->port_id = cpu_to_le16(bp->pf.port_id); 13249 req->phy_addr = phy_addr; 13250 req->reg_addr = cpu_to_le16(reg & 0x1f); 13251 if (mdio_phy_id_is_c45(phy_addr)) { 13252 req->cl45_mdio = 1; 13253 req->phy_addr = mdio_phy_id_prtad(phy_addr); 13254 req->dev_addr = mdio_phy_id_devad(phy_addr); 13255 req->reg_addr = cpu_to_le16(reg); 13256 } 13257 13258 resp = hwrm_req_hold(bp, req); 13259 rc = hwrm_req_send(bp, req); 13260 if (!rc) 13261 *val = le16_to_cpu(resp->reg_data); 13262 hwrm_req_drop(bp, req); 13263 return rc; 13264} 13265 13266static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg, 13267 u16 val) 13268{ 13269 struct hwrm_port_phy_mdio_write_input *req; 13270 int rc; 13271 13272 if (bp->hwrm_spec_code < 0x10a00) 13273 return -EOPNOTSUPP; 13274 13275 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE); 13276 if (rc) 13277 return rc; 13278 13279 req->port_id = cpu_to_le16(bp->pf.port_id); 13280 req->phy_addr = phy_addr; 13281 req->reg_addr = cpu_to_le16(reg & 0x1f); 13282 if (mdio_phy_id_is_c45(phy_addr)) { 13283 req->cl45_mdio = 1; 13284 req->phy_addr = mdio_phy_id_prtad(phy_addr); 13285 req->dev_addr = mdio_phy_id_devad(phy_addr); 13286 req->reg_addr = cpu_to_le16(reg); 13287 } 13288 req->reg_data = cpu_to_le16(val); 13289 13290 return hwrm_req_send(bp, req); 13291} 13292 13293/* netdev instance lock held */ 13294static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 13295{ 13296 struct mii_ioctl_data *mdio = if_mii(ifr); 13297 struct bnxt *bp = netdev_priv(dev); 13298 int rc; 13299 13300 switch (cmd) { 13301 case SIOCGMIIPHY: 13302 mdio->phy_id = bp->link_info.phy_addr; 13303 13304 fallthrough; 13305 case SIOCGMIIREG: { 13306 u16 mii_regval = 0; 13307 13308 if (!netif_running(dev)) 13309 return -EAGAIN; 13310 13311 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num, 13312 &mii_regval); 13313 mdio->val_out = mii_regval; 13314 return rc; 13315 } 13316 13317 case SIOCSMIIREG: 13318 if (!netif_running(dev)) 13319 return -EAGAIN; 13320 13321 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num, 13322 mdio->val_in); 13323 13324 default: 13325 /* do nothing */ 13326 break; 13327 } 13328 return -EOPNOTSUPP; 13329} 13330 13331static void bnxt_get_ring_stats(struct bnxt *bp, 13332 struct rtnl_link_stats64 *stats) 13333{ 13334 int i; 13335 13336 for (i = 0; i < bp->cp_nr_rings; i++) { 13337 struct bnxt_napi *bnapi = bp->bnapi[i]; 13338 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; 13339 u64 *sw = cpr->stats.sw_stats; 13340 13341 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 13342 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 13343 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 13344 13345 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 13346 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 13347 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 13348 13349 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 13350 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 13351 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 13352 13353 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 13354 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 13355 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 13356 13357 stats->rx_missed_errors += 13358 BNXT_GET_RING_STATS64(sw, rx_discard_pkts); 13359 13360 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 13361 13362 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts); 13363 13364 stats->rx_dropped += 13365 cpr->sw_stats->rx.rx_netpoll_discards + 13366 cpr->sw_stats->rx.rx_oom_discards; 13367 } 13368} 13369 13370static void bnxt_add_prev_stats(struct bnxt *bp, 13371 struct rtnl_link_stats64 *stats) 13372{ 13373 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev; 13374 13375 stats->rx_packets += prev_stats->rx_packets; 13376 stats->tx_packets += prev_stats->tx_packets; 13377 stats->rx_bytes += prev_stats->rx_bytes; 13378 stats->tx_bytes += prev_stats->tx_bytes; 13379 stats->rx_missed_errors += prev_stats->rx_missed_errors; 13380 stats->multicast += prev_stats->multicast; 13381 stats->rx_dropped += prev_stats->rx_dropped; 13382 stats->tx_dropped += prev_stats->tx_dropped; 13383} 13384 13385static void 13386bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 13387{ 13388 struct bnxt *bp = netdev_priv(dev); 13389 13390 set_bit(BNXT_STATE_READ_STATS, &bp->state); 13391 /* Make sure bnxt_close_nic() sees that we are reading stats before 13392 * we check the BNXT_STATE_OPEN flag. 13393 */ 13394 smp_mb__after_atomic(); 13395 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 13396 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 13397 *stats = bp->net_stats_prev; 13398 return; 13399 } 13400 13401 bnxt_get_ring_stats(bp, stats); 13402 bnxt_add_prev_stats(bp, stats); 13403 13404 if (bp->flags & BNXT_FLAG_PORT_STATS) { 13405 u64 *rx = bp->port_stats.sw_stats; 13406 u64 *tx = bp->port_stats.sw_stats + 13407 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8; 13408 13409 stats->rx_crc_errors = 13410 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); 13411 stats->rx_frame_errors = 13412 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames); 13413 stats->rx_length_errors = 13414 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + 13415 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + 13416 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames); 13417 stats->rx_errors = 13418 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + 13419 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames); 13420 stats->collisions = 13421 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions); 13422 stats->tx_fifo_errors = 13423 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); 13424 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); 13425 } 13426 clear_bit(BNXT_STATE_READ_STATS, &bp->state); 13427} 13428 13429static void bnxt_get_one_ring_err_stats(struct bnxt *bp, 13430 struct bnxt_total_ring_err_stats *stats, 13431 struct bnxt_cp_ring_info *cpr) 13432{ 13433 struct bnxt_sw_stats *sw_stats = cpr->sw_stats; 13434 u64 *hw_stats = cpr->stats.sw_stats; 13435 13436 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors; 13437 stats->rx_total_resets += sw_stats->rx.rx_resets; 13438 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors; 13439 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards; 13440 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards; 13441 stats->rx_total_ring_discards += 13442 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts); 13443 stats->tx_total_resets += sw_stats->tx.tx_resets; 13444 stats->tx_total_ring_discards += 13445 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts); 13446 stats->total_missed_irqs += sw_stats->cmn.missed_irqs; 13447} 13448 13449void bnxt_get_ring_err_stats(struct bnxt *bp, 13450 struct bnxt_total_ring_err_stats *stats) 13451{ 13452 int i; 13453 13454 for (i = 0; i < bp->cp_nr_rings; i++) 13455 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring); 13456} 13457 13458static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) 13459{ 13460 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 13461 struct net_device *dev = bp->dev; 13462 struct netdev_hw_addr *ha; 13463 u8 *haddr; 13464 int mc_count = 0; 13465 bool update = false; 13466 int off = 0; 13467 13468 netdev_for_each_mc_addr(ha, dev) { 13469 if (mc_count >= BNXT_MAX_MC_ADDRS) { 13470 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 13471 vnic->mc_list_count = 0; 13472 return false; 13473 } 13474 haddr = ha->addr; 13475 if (!ether_addr_equal(haddr, vnic->mc_list + off)) { 13476 memcpy(vnic->mc_list + off, haddr, ETH_ALEN); 13477 update = true; 13478 } 13479 off += ETH_ALEN; 13480 mc_count++; 13481 } 13482 if (mc_count) 13483 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 13484 13485 if (mc_count != vnic->mc_list_count) { 13486 vnic->mc_list_count = mc_count; 13487 update = true; 13488 } 13489 return update; 13490} 13491 13492static bool bnxt_uc_list_updated(struct bnxt *bp) 13493{ 13494 struct net_device *dev = bp->dev; 13495 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 13496 struct netdev_hw_addr *ha; 13497 int off = 0; 13498 13499 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1)) 13500 return true; 13501 13502 netdev_for_each_uc_addr(ha, dev) { 13503 if (!ether_addr_equal(ha->addr, vnic->uc_list + off)) 13504 return true; 13505 13506 off += ETH_ALEN; 13507 } 13508 return false; 13509} 13510 13511static void bnxt_set_rx_mode(struct net_device *dev) 13512{ 13513 struct bnxt *bp = netdev_priv(dev); 13514 struct bnxt_vnic_info *vnic; 13515 bool mc_update = false; 13516 bool uc_update; 13517 u32 mask; 13518 13519 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) 13520 return; 13521 13522 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 13523 mask = vnic->rx_mask; 13524 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS | 13525 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST | 13526 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST | 13527 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST); 13528 13529 if (dev->flags & IFF_PROMISC) 13530 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 13531 13532 uc_update = bnxt_uc_list_updated(bp); 13533 13534 if (dev->flags & IFF_BROADCAST) 13535 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST; 13536 if (dev->flags & IFF_ALLMULTI) { 13537 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 13538 vnic->mc_list_count = 0; 13539 } else if (dev->flags & IFF_MULTICAST) { 13540 mc_update = bnxt_mc_list_updated(bp, &mask); 13541 } 13542 13543 if (mask != vnic->rx_mask || uc_update || mc_update) { 13544 vnic->rx_mask = mask; 13545 13546 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 13547 } 13548} 13549 13550static int bnxt_cfg_rx_mode(struct bnxt *bp) 13551{ 13552 struct net_device *dev = bp->dev; 13553 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 13554 struct netdev_hw_addr *ha; 13555 int i, off = 0, rc; 13556 bool uc_update; 13557 13558 netif_addr_lock_bh(dev); 13559 uc_update = bnxt_uc_list_updated(bp); 13560 netif_addr_unlock_bh(dev); 13561 13562 if (!uc_update) 13563 goto skip_uc; 13564 13565 for (i = 1; i < vnic->uc_filter_count; i++) { 13566 struct bnxt_l2_filter *fltr = vnic->l2_filters[i]; 13567 13568 bnxt_hwrm_l2_filter_free(bp, fltr); 13569 bnxt_del_l2_filter(bp, fltr); 13570 } 13571 13572 vnic->uc_filter_count = 1; 13573 13574 netif_addr_lock_bh(dev); 13575 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) { 13576 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 13577 } else { 13578 netdev_for_each_uc_addr(ha, dev) { 13579 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN); 13580 off += ETH_ALEN; 13581 vnic->uc_filter_count++; 13582 } 13583 } 13584 netif_addr_unlock_bh(dev); 13585 13586 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) { 13587 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off); 13588 if (rc) { 13589 if (BNXT_VF(bp) && rc == -ENODEV) { 13590 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 13591 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n"); 13592 else 13593 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n"); 13594 rc = 0; 13595 } else { 13596 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc); 13597 } 13598 vnic->uc_filter_count = i; 13599 return rc; 13600 } 13601 } 13602 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 13603 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n"); 13604 13605skip_uc: 13606 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) && 13607 !bnxt_promisc_ok(bp)) 13608 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS; 13609 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 13610 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) { 13611 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n", 13612 rc); 13613 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST; 13614 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST; 13615 vnic->mc_list_count = 0; 13616 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0); 13617 } 13618 if (rc) 13619 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n", 13620 rc); 13621 13622 return rc; 13623} 13624 13625static bool bnxt_can_reserve_rings(struct bnxt *bp) 13626{ 13627#ifdef CONFIG_BNXT_SRIOV 13628 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) { 13629 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 13630 13631 /* No minimum rings were provisioned by the PF. Don't 13632 * reserve rings by default when device is down. 13633 */ 13634 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings) 13635 return true; 13636 13637 if (!netif_running(bp->dev)) 13638 return false; 13639 } 13640#endif 13641 return true; 13642} 13643 13644/* If the chip and firmware supports RFS */ 13645static bool bnxt_rfs_supported(struct bnxt *bp) 13646{ 13647 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 13648 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) 13649 return true; 13650 return false; 13651 } 13652 /* 212 firmware is broken for aRFS */ 13653 if (BNXT_FW_MAJ(bp) == 212) 13654 return false; 13655 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp)) 13656 return true; 13657 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) 13658 return true; 13659 return false; 13660} 13661 13662/* If runtime conditions support RFS */ 13663bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx) 13664{ 13665 struct bnxt_hw_rings hwr = {0}; 13666 int max_vnics, max_rss_ctxs; 13667 13668 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 13669 !BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 13670 return bnxt_rfs_supported(bp); 13671 13672 if (!bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings) 13673 return false; 13674 13675 hwr.grp = bp->rx_nr_rings; 13676 hwr.vnic = bnxt_get_total_vnics(bp, bp->rx_nr_rings); 13677 if (new_rss_ctx) 13678 hwr.vnic++; 13679 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr); 13680 max_vnics = bnxt_get_max_func_vnics(bp); 13681 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp); 13682 13683 if (hwr.vnic > max_vnics || hwr.rss_ctx > max_rss_ctxs) { 13684 if (bp->rx_nr_rings > 1) 13685 netdev_warn(bp->dev, 13686 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n", 13687 min(max_rss_ctxs - 1, max_vnics - 1)); 13688 return false; 13689 } 13690 13691 if (!BNXT_NEW_RM(bp)) 13692 return true; 13693 13694 /* Do not reduce VNIC and RSS ctx reservations. There is a FW 13695 * issue that will mess up the default VNIC if we reduce the 13696 * reservations. 13697 */ 13698 if (hwr.vnic <= bp->hw_resc.resv_vnics && 13699 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs) 13700 return true; 13701 13702 bnxt_hwrm_reserve_rings(bp, &hwr); 13703 if (hwr.vnic <= bp->hw_resc.resv_vnics && 13704 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs) 13705 return true; 13706 13707 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n"); 13708 hwr.vnic = 1; 13709 hwr.rss_ctx = 0; 13710 bnxt_hwrm_reserve_rings(bp, &hwr); 13711 return false; 13712} 13713 13714static netdev_features_t bnxt_fix_features(struct net_device *dev, 13715 netdev_features_t features) 13716{ 13717 struct bnxt *bp = netdev_priv(dev); 13718 netdev_features_t vlan_features; 13719 13720 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false)) 13721 features &= ~NETIF_F_NTUPLE; 13722 13723 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) 13724 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 13725 13726 if (!(features & NETIF_F_GRO)) 13727 features &= ~NETIF_F_GRO_HW; 13728 13729 if (features & NETIF_F_GRO_HW) 13730 features &= ~NETIF_F_LRO; 13731 13732 /* Both CTAG and STAG VLAN acceleration on the RX side have to be 13733 * turned on or off together. 13734 */ 13735 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX; 13736 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) { 13737 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13738 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 13739 else if (vlan_features) 13740 features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 13741 } 13742#ifdef CONFIG_BNXT_SRIOV 13743 if (BNXT_VF(bp) && bp->vf.vlan) 13744 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX; 13745#endif 13746 return features; 13747} 13748 13749static int bnxt_reinit_features(struct bnxt *bp, bool irq_re_init, 13750 bool link_re_init, u32 flags, bool update_tpa) 13751{ 13752 bnxt_close_nic(bp, irq_re_init, link_re_init); 13753 bp->flags = flags; 13754 if (update_tpa) 13755 bnxt_set_ring_params(bp); 13756 return bnxt_open_nic(bp, irq_re_init, link_re_init); 13757} 13758 13759static int bnxt_set_features(struct net_device *dev, netdev_features_t features) 13760{ 13761 bool update_tpa = false, update_ntuple = false; 13762 struct bnxt *bp = netdev_priv(dev); 13763 u32 flags = bp->flags; 13764 u32 changes; 13765 int rc = 0; 13766 bool re_init = false; 13767 13768 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; 13769 if (features & NETIF_F_GRO_HW) 13770 flags |= BNXT_FLAG_GRO; 13771 else if (features & NETIF_F_LRO) 13772 flags |= BNXT_FLAG_LRO; 13773 13774 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) 13775 flags &= ~BNXT_FLAG_TPA; 13776 13777 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX) 13778 flags |= BNXT_FLAG_STRIP_VLAN; 13779 13780 if (features & NETIF_F_NTUPLE) 13781 flags |= BNXT_FLAG_RFS; 13782 else 13783 bnxt_clear_usr_fltrs(bp, true); 13784 13785 changes = flags ^ bp->flags; 13786 if (changes & BNXT_FLAG_TPA) { 13787 update_tpa = true; 13788 if ((bp->flags & BNXT_FLAG_TPA) == 0 || 13789 (flags & BNXT_FLAG_TPA) == 0 || 13790 (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 13791 re_init = true; 13792 } 13793 13794 if (changes & ~BNXT_FLAG_TPA) 13795 re_init = true; 13796 13797 if (changes & BNXT_FLAG_RFS) 13798 update_ntuple = true; 13799 13800 if (flags != bp->flags) { 13801 u32 old_flags = bp->flags; 13802 13803 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 13804 bp->flags = flags; 13805 if (update_tpa) 13806 bnxt_set_ring_params(bp); 13807 return rc; 13808 } 13809 13810 if (update_ntuple) 13811 return bnxt_reinit_features(bp, true, false, flags, update_tpa); 13812 13813 if (re_init) 13814 return bnxt_reinit_features(bp, false, false, flags, update_tpa); 13815 13816 if (update_tpa) { 13817 bp->flags = flags; 13818 rc = bnxt_set_tpa(bp, 13819 (flags & BNXT_FLAG_TPA) ? 13820 true : false); 13821 if (rc) 13822 bp->flags = old_flags; 13823 } 13824 } 13825 return rc; 13826} 13827 13828static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off, 13829 u8 **nextp) 13830{ 13831 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off); 13832 struct hop_jumbo_hdr *jhdr; 13833 int hdr_count = 0; 13834 u8 *nexthdr; 13835 int start; 13836 13837 /* Check that there are at most 2 IPv6 extension headers, no 13838 * fragment header, and each is <= 64 bytes. 13839 */ 13840 start = nw_off + sizeof(*ip6h); 13841 nexthdr = &ip6h->nexthdr; 13842 while (ipv6_ext_hdr(*nexthdr)) { 13843 struct ipv6_opt_hdr *hp; 13844 int hdrlen; 13845 13846 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE || 13847 *nexthdr == NEXTHDR_FRAGMENT) 13848 return false; 13849 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data, 13850 skb_headlen(skb), NULL); 13851 if (!hp) 13852 return false; 13853 if (*nexthdr == NEXTHDR_AUTH) 13854 hdrlen = ipv6_authlen(hp); 13855 else 13856 hdrlen = ipv6_optlen(hp); 13857 13858 if (hdrlen > 64) 13859 return false; 13860 13861 /* The ext header may be a hop-by-hop header inserted for 13862 * big TCP purposes. This will be removed before sending 13863 * from NIC, so do not count it. 13864 */ 13865 if (*nexthdr == NEXTHDR_HOP) { 13866 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) 13867 goto increment_hdr; 13868 13869 jhdr = (struct hop_jumbo_hdr *)hp; 13870 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || 13871 jhdr->nexthdr != IPPROTO_TCP) 13872 goto increment_hdr; 13873 13874 goto next_hdr; 13875 } 13876increment_hdr: 13877 hdr_count++; 13878next_hdr: 13879 nexthdr = &hp->nexthdr; 13880 start += hdrlen; 13881 } 13882 if (nextp) { 13883 /* Caller will check inner protocol */ 13884 if (skb->encapsulation) { 13885 *nextp = nexthdr; 13886 return true; 13887 } 13888 *nextp = NULL; 13889 } 13890 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */ 13891 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP; 13892} 13893 13894/* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */ 13895static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb) 13896{ 13897 struct udphdr *uh = udp_hdr(skb); 13898 __be16 udp_port = uh->dest; 13899 13900 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port && 13901 udp_port != bp->vxlan_gpe_port) 13902 return false; 13903 if (skb->inner_protocol == htons(ETH_P_TEB)) { 13904 struct ethhdr *eh = inner_eth_hdr(skb); 13905 13906 switch (eh->h_proto) { 13907 case htons(ETH_P_IP): 13908 return true; 13909 case htons(ETH_P_IPV6): 13910 return bnxt_exthdr_check(bp, skb, 13911 skb_inner_network_offset(skb), 13912 NULL); 13913 } 13914 } else if (skb->inner_protocol == htons(ETH_P_IP)) { 13915 return true; 13916 } else if (skb->inner_protocol == htons(ETH_P_IPV6)) { 13917 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 13918 NULL); 13919 } 13920 return false; 13921} 13922 13923static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto) 13924{ 13925 switch (l4_proto) { 13926 case IPPROTO_UDP: 13927 return bnxt_udp_tunl_check(bp, skb); 13928 case IPPROTO_IPIP: 13929 return true; 13930 case IPPROTO_GRE: { 13931 switch (skb->inner_protocol) { 13932 default: 13933 return false; 13934 case htons(ETH_P_IP): 13935 return true; 13936 case htons(ETH_P_IPV6): 13937 fallthrough; 13938 } 13939 } 13940 case IPPROTO_IPV6: 13941 /* Check ext headers of inner ipv6 */ 13942 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb), 13943 NULL); 13944 } 13945 return false; 13946} 13947 13948static netdev_features_t bnxt_features_check(struct sk_buff *skb, 13949 struct net_device *dev, 13950 netdev_features_t features) 13951{ 13952 struct bnxt *bp = netdev_priv(dev); 13953 u8 *l4_proto; 13954 13955 features = vlan_features_check(skb, features); 13956 switch (vlan_get_protocol(skb)) { 13957 case htons(ETH_P_IP): 13958 if (!skb->encapsulation) 13959 return features; 13960 l4_proto = &ip_hdr(skb)->protocol; 13961 if (bnxt_tunl_check(bp, skb, *l4_proto)) 13962 return features; 13963 break; 13964 case htons(ETH_P_IPV6): 13965 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb), 13966 &l4_proto)) 13967 break; 13968 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto)) 13969 return features; 13970 break; 13971 } 13972 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 13973} 13974 13975int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, 13976 u32 *reg_buf) 13977{ 13978 struct hwrm_dbg_read_direct_output *resp; 13979 struct hwrm_dbg_read_direct_input *req; 13980 __le32 *dbg_reg_buf; 13981 dma_addr_t mapping; 13982 int rc, i; 13983 13984 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT); 13985 if (rc) 13986 return rc; 13987 13988 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4, 13989 &mapping); 13990 if (!dbg_reg_buf) { 13991 rc = -ENOMEM; 13992 goto dbg_rd_reg_exit; 13993 } 13994 13995 req->host_dest_addr = cpu_to_le64(mapping); 13996 13997 resp = hwrm_req_hold(bp, req); 13998 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR); 13999 req->read_len32 = cpu_to_le32(num_words); 14000 14001 rc = hwrm_req_send(bp, req); 14002 if (rc || resp->error_code) { 14003 rc = -EIO; 14004 goto dbg_rd_reg_exit; 14005 } 14006 for (i = 0; i < num_words; i++) 14007 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]); 14008 14009dbg_rd_reg_exit: 14010 hwrm_req_drop(bp, req); 14011 return rc; 14012} 14013 14014static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type, 14015 u32 ring_id, u32 *prod, u32 *cons) 14016{ 14017 struct hwrm_dbg_ring_info_get_output *resp; 14018 struct hwrm_dbg_ring_info_get_input *req; 14019 int rc; 14020 14021 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET); 14022 if (rc) 14023 return rc; 14024 14025 req->ring_type = ring_type; 14026 req->fw_ring_id = cpu_to_le32(ring_id); 14027 resp = hwrm_req_hold(bp, req); 14028 rc = hwrm_req_send(bp, req); 14029 if (!rc) { 14030 *prod = le32_to_cpu(resp->producer_index); 14031 *cons = le32_to_cpu(resp->consumer_index); 14032 } 14033 hwrm_req_drop(bp, req); 14034 return rc; 14035} 14036 14037static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi) 14038{ 14039 struct bnxt_tx_ring_info *txr; 14040 int i = bnapi->index, j; 14041 14042 bnxt_for_each_napi_tx(j, bnapi, txr) 14043 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n", 14044 i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod, 14045 txr->tx_cons); 14046} 14047 14048static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi) 14049{ 14050 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 14051 int i = bnapi->index; 14052 14053 if (!rxr) 14054 return; 14055 14056 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n", 14057 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod, 14058 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod, 14059 rxr->rx_sw_agg_prod); 14060} 14061 14062static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi) 14063{ 14064 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring, *cpr2; 14065 int i = bnapi->index, j; 14066 14067 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n", 14068 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons); 14069 for (j = 0; j < cpr->cp_ring_count; j++) { 14070 cpr2 = &cpr->cp_ring_arr[j]; 14071 if (!cpr2->bnapi) 14072 continue; 14073 netdev_info(bnapi->bp->dev, "[%d.%d]: cp{fw_ring: %d raw_cons: %x}\n", 14074 i, j, cpr2->cp_ring_struct.fw_ring_id, 14075 cpr2->cp_raw_cons); 14076 } 14077} 14078 14079static void bnxt_dbg_dump_states(struct bnxt *bp) 14080{ 14081 int i; 14082 struct bnxt_napi *bnapi; 14083 14084 for (i = 0; i < bp->cp_nr_rings; i++) { 14085 bnapi = bp->bnapi[i]; 14086 if (netif_msg_drv(bp)) { 14087 bnxt_dump_tx_sw_state(bnapi); 14088 bnxt_dump_rx_sw_state(bnapi); 14089 bnxt_dump_cp_sw_state(bnapi); 14090 } 14091 } 14092} 14093 14094static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr) 14095{ 14096 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr]; 14097 struct hwrm_ring_reset_input *req; 14098 struct bnxt_napi *bnapi = rxr->bnapi; 14099 struct bnxt_cp_ring_info *cpr; 14100 u16 cp_ring_id; 14101 int rc; 14102 14103 rc = hwrm_req_init(bp, req, HWRM_RING_RESET); 14104 if (rc) 14105 return rc; 14106 14107 cpr = &bnapi->cp_ring; 14108 cp_ring_id = cpr->cp_ring_struct.fw_ring_id; 14109 req->cmpl_ring = cpu_to_le16(cp_ring_id); 14110 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP; 14111 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id); 14112 return hwrm_req_send_silent(bp, req); 14113} 14114 14115static void bnxt_reset_task(struct bnxt *bp, bool silent) 14116{ 14117 if (!silent) 14118 bnxt_dbg_dump_states(bp); 14119 if (netif_running(bp->dev)) { 14120 bnxt_close_nic(bp, !silent, false); 14121 bnxt_open_nic(bp, !silent, false); 14122 } 14123} 14124 14125static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue) 14126{ 14127 struct bnxt *bp = netdev_priv(dev); 14128 14129 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n"); 14130 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT); 14131} 14132 14133static void bnxt_fw_health_check(struct bnxt *bp) 14134{ 14135 struct bnxt_fw_health *fw_health = bp->fw_health; 14136 struct pci_dev *pdev = bp->pdev; 14137 u32 val; 14138 14139 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 14140 return; 14141 14142 /* Make sure it is enabled before checking the tmr_counter. */ 14143 smp_rmb(); 14144 if (fw_health->tmr_counter) { 14145 fw_health->tmr_counter--; 14146 return; 14147 } 14148 14149 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 14150 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) { 14151 fw_health->arrests++; 14152 goto fw_reset; 14153 } 14154 14155 fw_health->last_fw_heartbeat = val; 14156 14157 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 14158 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) { 14159 fw_health->discoveries++; 14160 goto fw_reset; 14161 } 14162 14163 fw_health->tmr_counter = fw_health->tmr_multiplier; 14164 return; 14165 14166fw_reset: 14167 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT); 14168} 14169 14170static void bnxt_timer(struct timer_list *t) 14171{ 14172 struct bnxt *bp = timer_container_of(bp, t, timer); 14173 struct net_device *dev = bp->dev; 14174 14175 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state)) 14176 return; 14177 14178 if (atomic_read(&bp->intr_sem) != 0) 14179 goto bnxt_restart_timer; 14180 14181 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) 14182 bnxt_fw_health_check(bp); 14183 14184 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks) 14185 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT); 14186 14187 if (bnxt_tc_flower_enabled(bp)) 14188 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT); 14189 14190#ifdef CONFIG_RFS_ACCEL 14191 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) 14192 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 14193#endif /*CONFIG_RFS_ACCEL*/ 14194 14195 if (bp->link_info.phy_retry) { 14196 if (time_after(jiffies, bp->link_info.phy_retry_expires)) { 14197 bp->link_info.phy_retry = false; 14198 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n"); 14199 } else { 14200 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT); 14201 } 14202 } 14203 14204 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state)) 14205 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT); 14206 14207 if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev)) 14208 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT); 14209 14210bnxt_restart_timer: 14211 mod_timer(&bp->timer, jiffies + bp->current_interval); 14212} 14213 14214static void bnxt_lock_sp(struct bnxt *bp) 14215{ 14216 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK 14217 * set. If the device is being closed, bnxt_close() may be holding 14218 * netdev instance lock and waiting for BNXT_STATE_IN_SP_TASK to clear. 14219 * So we must clear BNXT_STATE_IN_SP_TASK before holding netdev 14220 * instance lock. 14221 */ 14222 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 14223 netdev_lock(bp->dev); 14224} 14225 14226static void bnxt_unlock_sp(struct bnxt *bp) 14227{ 14228 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 14229 netdev_unlock(bp->dev); 14230} 14231 14232/* Only called from bnxt_sp_task() */ 14233static void bnxt_reset(struct bnxt *bp, bool silent) 14234{ 14235 bnxt_lock_sp(bp); 14236 if (test_bit(BNXT_STATE_OPEN, &bp->state)) 14237 bnxt_reset_task(bp, silent); 14238 bnxt_unlock_sp(bp); 14239} 14240 14241/* Only called from bnxt_sp_task() */ 14242static void bnxt_rx_ring_reset(struct bnxt *bp) 14243{ 14244 int i; 14245 14246 bnxt_lock_sp(bp); 14247 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 14248 bnxt_unlock_sp(bp); 14249 return; 14250 } 14251 /* Disable and flush TPA before resetting the RX ring */ 14252 if (bp->flags & BNXT_FLAG_TPA) 14253 bnxt_set_tpa(bp, false); 14254 for (i = 0; i < bp->rx_nr_rings; i++) { 14255 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i]; 14256 struct bnxt_cp_ring_info *cpr; 14257 int rc; 14258 14259 if (!rxr->bnapi->in_reset) 14260 continue; 14261 14262 rc = bnxt_hwrm_rx_ring_reset(bp, i); 14263 if (rc) { 14264 if (rc == -EINVAL || rc == -EOPNOTSUPP) 14265 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n"); 14266 else 14267 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n", 14268 rc); 14269 bnxt_reset_task(bp, true); 14270 break; 14271 } 14272 bnxt_free_one_rx_ring_skbs(bp, rxr); 14273 rxr->rx_prod = 0; 14274 rxr->rx_agg_prod = 0; 14275 rxr->rx_sw_agg_prod = 0; 14276 rxr->rx_next_cons = 0; 14277 rxr->bnapi->in_reset = false; 14278 bnxt_alloc_one_rx_ring(bp, i); 14279 cpr = &rxr->bnapi->cp_ring; 14280 cpr->sw_stats->rx.rx_resets++; 14281 if (bp->flags & BNXT_FLAG_AGG_RINGS) 14282 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 14283 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 14284 } 14285 if (bp->flags & BNXT_FLAG_TPA) 14286 bnxt_set_tpa(bp, true); 14287 bnxt_unlock_sp(bp); 14288} 14289 14290static void bnxt_fw_fatal_close(struct bnxt *bp) 14291{ 14292 bnxt_tx_disable(bp); 14293 bnxt_disable_napi(bp); 14294 bnxt_disable_int_sync(bp); 14295 bnxt_free_irq(bp); 14296 bnxt_clear_int_mode(bp); 14297 pci_disable_device(bp->pdev); 14298} 14299 14300static void bnxt_fw_reset_close(struct bnxt *bp) 14301{ 14302 /* When firmware is in fatal state, quiesce device and disable 14303 * bus master to prevent any potential bad DMAs before freeing 14304 * kernel memory. 14305 */ 14306 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { 14307 u16 val = 0; 14308 14309 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 14310 if (val == 0xffff) 14311 bp->fw_reset_min_dsecs = 0; 14312 bnxt_fw_fatal_close(bp); 14313 } 14314 __bnxt_close_nic(bp, true, false); 14315 bnxt_vf_reps_free(bp); 14316 bnxt_clear_int_mode(bp); 14317 bnxt_hwrm_func_drv_unrgtr(bp); 14318 if (pci_is_enabled(bp->pdev)) 14319 pci_disable_device(bp->pdev); 14320 bnxt_free_ctx_mem(bp, false); 14321} 14322 14323static bool is_bnxt_fw_ok(struct bnxt *bp) 14324{ 14325 struct bnxt_fw_health *fw_health = bp->fw_health; 14326 bool no_heartbeat = false, has_reset = false; 14327 u32 val; 14328 14329 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); 14330 if (val == fw_health->last_fw_heartbeat) 14331 no_heartbeat = true; 14332 14333 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 14334 if (val != fw_health->last_fw_reset_cnt) 14335 has_reset = true; 14336 14337 if (!no_heartbeat && has_reset) 14338 return true; 14339 14340 return false; 14341} 14342 14343/* netdev instance lock is acquired before calling this function */ 14344static void bnxt_force_fw_reset(struct bnxt *bp) 14345{ 14346 struct bnxt_fw_health *fw_health = bp->fw_health; 14347 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 14348 u32 wait_dsecs; 14349 14350 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 14351 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) 14352 return; 14353 14354 /* we have to serialize with bnxt_refclk_read()*/ 14355 if (ptp) { 14356 unsigned long flags; 14357 14358 write_seqlock_irqsave(&ptp->ptp_lock, flags); 14359 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14360 write_sequnlock_irqrestore(&ptp->ptp_lock, flags); 14361 } else { 14362 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14363 } 14364 bnxt_fw_reset_close(bp); 14365 wait_dsecs = fw_health->master_func_wait_dsecs; 14366 if (fw_health->primary) { 14367 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) 14368 wait_dsecs = 0; 14369 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 14370 } else { 14371 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; 14372 wait_dsecs = fw_health->normal_func_wait_dsecs; 14373 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 14374 } 14375 14376 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; 14377 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; 14378 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 14379} 14380 14381void bnxt_fw_exception(struct bnxt *bp) 14382{ 14383 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); 14384 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 14385 bnxt_ulp_stop(bp); 14386 bnxt_lock_sp(bp); 14387 bnxt_force_fw_reset(bp); 14388 bnxt_unlock_sp(bp); 14389} 14390 14391/* Returns the number of registered VFs, or 1 if VF configuration is pending, or 14392 * < 0 on error. 14393 */ 14394static int bnxt_get_registered_vfs(struct bnxt *bp) 14395{ 14396#ifdef CONFIG_BNXT_SRIOV 14397 int rc; 14398 14399 if (!BNXT_PF(bp)) 14400 return 0; 14401 14402 rc = bnxt_hwrm_func_qcfg(bp); 14403 if (rc) { 14404 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc); 14405 return rc; 14406 } 14407 if (bp->pf.registered_vfs) 14408 return bp->pf.registered_vfs; 14409 if (bp->sriov_cfg) 14410 return 1; 14411#endif 14412 return 0; 14413} 14414 14415void bnxt_fw_reset(struct bnxt *bp) 14416{ 14417 bnxt_ulp_stop(bp); 14418 bnxt_lock_sp(bp); 14419 if (test_bit(BNXT_STATE_OPEN, &bp->state) && 14420 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 14421 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; 14422 int n = 0, tmo; 14423 14424 /* we have to serialize with bnxt_refclk_read()*/ 14425 if (ptp) { 14426 unsigned long flags; 14427 14428 write_seqlock_irqsave(&ptp->ptp_lock, flags); 14429 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14430 write_sequnlock_irqrestore(&ptp->ptp_lock, flags); 14431 } else { 14432 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14433 } 14434 if (bp->pf.active_vfs && 14435 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) 14436 n = bnxt_get_registered_vfs(bp); 14437 if (n < 0) { 14438 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n", 14439 n); 14440 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 14441 netif_close(bp->dev); 14442 goto fw_reset_exit; 14443 } else if (n > 0) { 14444 u16 vf_tmo_dsecs = n * 10; 14445 14446 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs) 14447 bp->fw_reset_max_dsecs = vf_tmo_dsecs; 14448 bp->fw_reset_state = 14449 BNXT_FW_RESET_STATE_POLL_VF; 14450 bnxt_queue_fw_reset_work(bp, HZ / 10); 14451 goto fw_reset_exit; 14452 } 14453 bnxt_fw_reset_close(bp); 14454 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 14455 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 14456 tmo = HZ / 10; 14457 } else { 14458 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 14459 tmo = bp->fw_reset_min_dsecs * HZ / 10; 14460 } 14461 bnxt_queue_fw_reset_work(bp, tmo); 14462 } 14463fw_reset_exit: 14464 bnxt_unlock_sp(bp); 14465} 14466 14467static void bnxt_chk_missed_irq(struct bnxt *bp) 14468{ 14469 int i; 14470 14471 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 14472 return; 14473 14474 for (i = 0; i < bp->cp_nr_rings; i++) { 14475 struct bnxt_napi *bnapi = bp->bnapi[i]; 14476 struct bnxt_cp_ring_info *cpr; 14477 u32 fw_ring_id; 14478 int j; 14479 14480 if (!bnapi) 14481 continue; 14482 14483 cpr = &bnapi->cp_ring; 14484 for (j = 0; j < cpr->cp_ring_count; j++) { 14485 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j]; 14486 u32 val[2]; 14487 14488 if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2)) 14489 continue; 14490 14491 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) { 14492 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons; 14493 continue; 14494 } 14495 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id; 14496 bnxt_dbg_hwrm_ring_info_get(bp, 14497 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL, 14498 fw_ring_id, &val[0], &val[1]); 14499 cpr->sw_stats->cmn.missed_irqs++; 14500 } 14501 } 14502} 14503 14504static void bnxt_cfg_ntp_filters(struct bnxt *); 14505 14506static void bnxt_init_ethtool_link_settings(struct bnxt *bp) 14507{ 14508 struct bnxt_link_info *link_info = &bp->link_info; 14509 14510 if (BNXT_AUTO_MODE(link_info->auto_mode)) { 14511 link_info->autoneg = BNXT_AUTONEG_SPEED; 14512 if (bp->hwrm_spec_code >= 0x10201) { 14513 if (link_info->auto_pause_setting & 14514 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE) 14515 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 14516 } else { 14517 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL; 14518 } 14519 bnxt_set_auto_speed(link_info); 14520 } else { 14521 bnxt_set_force_speed(link_info); 14522 link_info->req_duplex = link_info->duplex_setting; 14523 } 14524 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) 14525 link_info->req_flow_ctrl = 14526 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH; 14527 else 14528 link_info->req_flow_ctrl = link_info->force_pause_setting; 14529} 14530 14531static void bnxt_fw_echo_reply(struct bnxt *bp) 14532{ 14533 struct bnxt_fw_health *fw_health = bp->fw_health; 14534 struct hwrm_func_echo_response_input *req; 14535 int rc; 14536 14537 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE); 14538 if (rc) 14539 return; 14540 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1); 14541 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2); 14542 hwrm_req_send(bp, req); 14543} 14544 14545static void bnxt_ulp_restart(struct bnxt *bp) 14546{ 14547 bnxt_ulp_stop(bp); 14548 bnxt_ulp_start(bp, 0); 14549} 14550 14551static void bnxt_sp_task(struct work_struct *work) 14552{ 14553 struct bnxt *bp = container_of(work, struct bnxt, sp_task); 14554 14555 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 14556 smp_mb__after_atomic(); 14557 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { 14558 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 14559 return; 14560 } 14561 14562 if (test_and_clear_bit(BNXT_RESTART_ULP_SP_EVENT, &bp->sp_event)) { 14563 bnxt_ulp_restart(bp); 14564 bnxt_reenable_sriov(bp); 14565 } 14566 14567 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event)) 14568 bnxt_cfg_rx_mode(bp); 14569 14570 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event)) 14571 bnxt_cfg_ntp_filters(bp); 14572 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event)) 14573 bnxt_hwrm_exec_fwd_req(bp); 14574 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event)) 14575 netdev_info(bp->dev, "Receive PF driver unload event!\n"); 14576 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) { 14577 bnxt_hwrm_port_qstats(bp, 0); 14578 bnxt_hwrm_port_qstats_ext(bp, 0); 14579 bnxt_accumulate_all_stats(bp); 14580 } 14581 14582 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) { 14583 int rc; 14584 14585 mutex_lock(&bp->link_lock); 14586 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, 14587 &bp->sp_event)) 14588 bnxt_hwrm_phy_qcaps(bp); 14589 14590 rc = bnxt_update_link(bp, true); 14591 if (rc) 14592 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n", 14593 rc); 14594 14595 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, 14596 &bp->sp_event)) 14597 bnxt_init_ethtool_link_settings(bp); 14598 mutex_unlock(&bp->link_lock); 14599 } 14600 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) { 14601 int rc; 14602 14603 mutex_lock(&bp->link_lock); 14604 rc = bnxt_update_phy_setting(bp); 14605 mutex_unlock(&bp->link_lock); 14606 if (rc) { 14607 netdev_warn(bp->dev, "update phy settings retry failed\n"); 14608 } else { 14609 bp->link_info.phy_retry = false; 14610 netdev_info(bp->dev, "update phy settings retry succeeded\n"); 14611 } 14612 } 14613 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) { 14614 mutex_lock(&bp->link_lock); 14615 bnxt_get_port_module_status(bp); 14616 mutex_unlock(&bp->link_lock); 14617 } 14618 14619 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event)) 14620 bnxt_tc_flow_stats_work(bp); 14621 14622 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event)) 14623 bnxt_chk_missed_irq(bp); 14624 14625 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event)) 14626 bnxt_fw_echo_reply(bp); 14627 14628 if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event)) 14629 bnxt_hwmon_notify_event(bp); 14630 14631 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They 14632 * must be the last functions to be called before exiting. 14633 */ 14634 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event)) 14635 bnxt_reset(bp, false); 14636 14637 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event)) 14638 bnxt_reset(bp, true); 14639 14640 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event)) 14641 bnxt_rx_ring_reset(bp); 14642 14643 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { 14644 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || 14645 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) 14646 bnxt_devlink_health_fw_report(bp); 14647 else 14648 bnxt_fw_reset(bp); 14649 } 14650 14651 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { 14652 if (!is_bnxt_fw_ok(bp)) 14653 bnxt_devlink_health_fw_report(bp); 14654 } 14655 14656 smp_mb__before_atomic(); 14657 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); 14658} 14659 14660static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 14661 int *max_cp); 14662 14663/* Under netdev instance lock */ 14664int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, 14665 int tx_xdp) 14666{ 14667 int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp; 14668 struct bnxt_hw_rings hwr = {0}; 14669 int rx_rings = rx; 14670 int rc; 14671 14672 if (tcs) 14673 tx_sets = tcs; 14674 14675 _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp); 14676 14677 if (max_rx < rx_rings) 14678 return -ENOMEM; 14679 14680 if (bp->flags & BNXT_FLAG_AGG_RINGS) 14681 rx_rings <<= 1; 14682 14683 hwr.rx = rx_rings; 14684 hwr.tx = tx * tx_sets + tx_xdp; 14685 if (max_tx < hwr.tx) 14686 return -ENOMEM; 14687 14688 hwr.vnic = bnxt_get_total_vnics(bp, rx); 14689 14690 tx_cp = __bnxt_num_tx_to_cp(bp, hwr.tx, tx_sets, tx_xdp); 14691 hwr.cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx; 14692 if (max_cp < hwr.cp) 14693 return -ENOMEM; 14694 hwr.stat = hwr.cp; 14695 if (BNXT_NEW_RM(bp)) { 14696 hwr.cp += bnxt_get_ulp_msix_num_in_use(bp); 14697 hwr.stat += bnxt_get_ulp_stat_ctxs_in_use(bp); 14698 hwr.grp = rx; 14699 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr); 14700 } 14701 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 14702 hwr.cp_p5 = hwr.tx + rx; 14703 rc = bnxt_hwrm_check_rings(bp, &hwr); 14704 if (!rc && pci_msix_can_alloc_dyn(bp->pdev)) { 14705 if (!bnxt_ulp_registered(bp->edev)) { 14706 hwr.cp += bnxt_get_ulp_msix_num(bp); 14707 hwr.cp = min_t(int, hwr.cp, bnxt_get_max_func_irqs(bp)); 14708 } 14709 if (hwr.cp > bp->total_irqs) { 14710 int total_msix = bnxt_change_msix(bp, hwr.cp); 14711 14712 if (total_msix < hwr.cp) { 14713 netdev_warn(bp->dev, "Unable to allocate %d MSIX vectors, maximum available %d\n", 14714 hwr.cp, total_msix); 14715 rc = -ENOSPC; 14716 } 14717 } 14718 } 14719 return rc; 14720} 14721 14722static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev) 14723{ 14724 if (bp->bar2) { 14725 pci_iounmap(pdev, bp->bar2); 14726 bp->bar2 = NULL; 14727 } 14728 14729 if (bp->bar1) { 14730 pci_iounmap(pdev, bp->bar1); 14731 bp->bar1 = NULL; 14732 } 14733 14734 if (bp->bar0) { 14735 pci_iounmap(pdev, bp->bar0); 14736 bp->bar0 = NULL; 14737 } 14738} 14739 14740static void bnxt_cleanup_pci(struct bnxt *bp) 14741{ 14742 bnxt_unmap_bars(bp, bp->pdev); 14743 pci_release_regions(bp->pdev); 14744 if (pci_is_enabled(bp->pdev)) 14745 pci_disable_device(bp->pdev); 14746} 14747 14748static void bnxt_init_dflt_coal(struct bnxt *bp) 14749{ 14750 struct bnxt_coal_cap *coal_cap = &bp->coal_cap; 14751 struct bnxt_coal *coal; 14752 u16 flags = 0; 14753 14754 if (coal_cap->cmpl_params & 14755 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET) 14756 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET; 14757 14758 /* Tick values in micro seconds. 14759 * 1 coal_buf x bufs_per_record = 1 completion record. 14760 */ 14761 coal = &bp->rx_coal; 14762 coal->coal_ticks = 10; 14763 coal->coal_bufs = 30; 14764 coal->coal_ticks_irq = 1; 14765 coal->coal_bufs_irq = 2; 14766 coal->idle_thresh = 50; 14767 coal->bufs_per_record = 2; 14768 coal->budget = 64; /* NAPI budget */ 14769 coal->flags = flags; 14770 14771 coal = &bp->tx_coal; 14772 coal->coal_ticks = 28; 14773 coal->coal_bufs = 30; 14774 coal->coal_ticks_irq = 2; 14775 coal->coal_bufs_irq = 2; 14776 coal->bufs_per_record = 1; 14777 coal->flags = flags; 14778 14779 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS; 14780} 14781 14782/* FW that pre-reserves 1 VNIC per function */ 14783static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp) 14784{ 14785 u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp); 14786 14787 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 14788 (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18))) 14789 return true; 14790 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 14791 (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172))) 14792 return true; 14793 return false; 14794} 14795 14796static void bnxt_hwrm_pfcwd_qcaps(struct bnxt *bp) 14797{ 14798 struct hwrm_queue_pfcwd_timeout_qcaps_output *resp; 14799 struct hwrm_queue_pfcwd_timeout_qcaps_input *req; 14800 int rc; 14801 14802 bp->max_pfcwd_tmo_ms = 0; 14803 rc = hwrm_req_init(bp, req, HWRM_QUEUE_PFCWD_TIMEOUT_QCAPS); 14804 if (rc) 14805 return; 14806 resp = hwrm_req_hold(bp, req); 14807 rc = hwrm_req_send_silent(bp, req); 14808 if (!rc) 14809 bp->max_pfcwd_tmo_ms = le16_to_cpu(resp->max_pfcwd_timeout); 14810 hwrm_req_drop(bp, req); 14811} 14812 14813static int bnxt_fw_init_one_p1(struct bnxt *bp) 14814{ 14815 int rc; 14816 14817 bp->fw_cap = 0; 14818 rc = bnxt_hwrm_ver_get(bp); 14819 /* FW may be unresponsive after FLR. FLR must complete within 100 msec 14820 * so wait before continuing with recovery. 14821 */ 14822 if (rc) 14823 msleep(100); 14824 bnxt_try_map_fw_health_reg(bp); 14825 if (rc) { 14826 rc = bnxt_try_recover_fw(bp); 14827 if (rc) 14828 return rc; 14829 rc = bnxt_hwrm_ver_get(bp); 14830 if (rc) 14831 return rc; 14832 } 14833 14834 bnxt_nvm_cfg_ver_get(bp); 14835 14836 rc = bnxt_hwrm_func_reset(bp); 14837 if (rc) 14838 return -ENODEV; 14839 14840 bnxt_hwrm_fw_set_time(bp); 14841 return 0; 14842} 14843 14844static int bnxt_fw_init_one_p2(struct bnxt *bp) 14845{ 14846 int rc; 14847 14848 /* Get the MAX capabilities for this function */ 14849 rc = bnxt_hwrm_func_qcaps(bp); 14850 if (rc) { 14851 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n", 14852 rc); 14853 return -ENODEV; 14854 } 14855 14856 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp); 14857 if (rc) 14858 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n", 14859 rc); 14860 14861 if (bnxt_alloc_fw_health(bp)) { 14862 netdev_warn(bp->dev, "no memory for firmware error recovery\n"); 14863 } else { 14864 rc = bnxt_hwrm_error_recovery_qcfg(bp); 14865 if (rc) 14866 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n", 14867 rc); 14868 } 14869 14870 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); 14871 if (rc) 14872 return -ENODEV; 14873 14874 rc = bnxt_alloc_crash_dump_mem(bp); 14875 if (rc) 14876 netdev_warn(bp->dev, "crash dump mem alloc failure rc: %d\n", 14877 rc); 14878 if (!rc) { 14879 rc = bnxt_hwrm_crash_dump_mem_cfg(bp); 14880 if (rc) { 14881 bnxt_free_crash_dump_mem(bp); 14882 netdev_warn(bp->dev, 14883 "hwrm crash dump mem failure rc: %d\n", rc); 14884 } 14885 } 14886 14887 if (bnxt_fw_pre_resv_vnics(bp)) 14888 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS; 14889 14890 bnxt_hwrm_pfcwd_qcaps(bp); 14891 bnxt_hwrm_func_qcfg(bp); 14892 bnxt_hwrm_vnic_qcaps(bp); 14893 bnxt_hwrm_port_led_qcaps(bp); 14894 bnxt_ethtool_init(bp); 14895 if (bp->fw_cap & BNXT_FW_CAP_PTP) 14896 __bnxt_hwrm_ptp_qcfg(bp); 14897 bnxt_dcb_init(bp); 14898 bnxt_hwmon_init(bp); 14899 return 0; 14900} 14901 14902static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp) 14903{ 14904 bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP; 14905 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 | 14906 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 | 14907 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 | 14908 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6; 14909 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA) 14910 bp->rss_hash_delta = bp->rss_hash_cfg; 14911 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) { 14912 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP; 14913 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 | 14914 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6; 14915 } 14916} 14917 14918static void bnxt_set_dflt_rfs(struct bnxt *bp) 14919{ 14920 struct net_device *dev = bp->dev; 14921 14922 dev->hw_features &= ~NETIF_F_NTUPLE; 14923 dev->features &= ~NETIF_F_NTUPLE; 14924 bp->flags &= ~BNXT_FLAG_RFS; 14925 if (bnxt_rfs_supported(bp)) { 14926 dev->hw_features |= NETIF_F_NTUPLE; 14927 if (bnxt_rfs_capable(bp, false)) { 14928 bp->flags |= BNXT_FLAG_RFS; 14929 dev->features |= NETIF_F_NTUPLE; 14930 } 14931 } 14932} 14933 14934static void bnxt_fw_init_one_p3(struct bnxt *bp) 14935{ 14936 struct pci_dev *pdev = bp->pdev; 14937 14938 bnxt_set_dflt_rss_hash_type(bp); 14939 bnxt_set_dflt_rfs(bp); 14940 14941 bnxt_get_wol_settings(bp); 14942 if (bp->flags & BNXT_FLAG_WOL_CAP) 14943 device_set_wakeup_enable(&pdev->dev, bp->wol); 14944 else 14945 device_set_wakeup_capable(&pdev->dev, false); 14946 14947 bnxt_hwrm_set_cache_line_size(bp, cache_line_size()); 14948 bnxt_hwrm_coal_params_qcaps(bp); 14949} 14950 14951static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt); 14952 14953int bnxt_fw_init_one(struct bnxt *bp) 14954{ 14955 int rc; 14956 14957 rc = bnxt_fw_init_one_p1(bp); 14958 if (rc) { 14959 netdev_err(bp->dev, "Firmware init phase 1 failed\n"); 14960 return rc; 14961 } 14962 rc = bnxt_fw_init_one_p2(bp); 14963 if (rc) { 14964 netdev_err(bp->dev, "Firmware init phase 2 failed\n"); 14965 return rc; 14966 } 14967 rc = bnxt_probe_phy(bp, false); 14968 if (rc) 14969 return rc; 14970 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false); 14971 if (rc) 14972 return rc; 14973 14974 bnxt_fw_init_one_p3(bp); 14975 return 0; 14976} 14977 14978static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx) 14979{ 14980 struct bnxt_fw_health *fw_health = bp->fw_health; 14981 u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; 14982 u32 val = fw_health->fw_reset_seq_vals[reg_idx]; 14983 u32 reg_type, reg_off, delay_msecs; 14984 14985 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; 14986 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); 14987 reg_off = BNXT_FW_HEALTH_REG_OFF(reg); 14988 switch (reg_type) { 14989 case BNXT_FW_HEALTH_REG_TYPE_CFG: 14990 pci_write_config_dword(bp->pdev, reg_off, val); 14991 break; 14992 case BNXT_FW_HEALTH_REG_TYPE_GRC: 14993 writel(reg_off & BNXT_GRC_BASE_MASK, 14994 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4); 14995 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; 14996 fallthrough; 14997 case BNXT_FW_HEALTH_REG_TYPE_BAR0: 14998 writel(val, bp->bar0 + reg_off); 14999 break; 15000 case BNXT_FW_HEALTH_REG_TYPE_BAR1: 15001 writel(val, bp->bar1 + reg_off); 15002 break; 15003 } 15004 if (delay_msecs) { 15005 pci_read_config_dword(bp->pdev, 0, &val); 15006 msleep(delay_msecs); 15007 } 15008} 15009 15010bool bnxt_hwrm_reset_permitted(struct bnxt *bp) 15011{ 15012 struct hwrm_func_qcfg_output *resp; 15013 struct hwrm_func_qcfg_input *req; 15014 bool result = true; /* firmware will enforce if unknown */ 15015 15016 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF) 15017 return result; 15018 15019 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG)) 15020 return result; 15021 15022 req->fid = cpu_to_le16(0xffff); 15023 resp = hwrm_req_hold(bp, req); 15024 if (!hwrm_req_send(bp, req)) 15025 result = !!(le16_to_cpu(resp->flags) & 15026 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED); 15027 hwrm_req_drop(bp, req); 15028 return result; 15029} 15030 15031static void bnxt_reset_all(struct bnxt *bp) 15032{ 15033 struct bnxt_fw_health *fw_health = bp->fw_health; 15034 int i, rc; 15035 15036 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 15037 bnxt_fw_reset_via_optee(bp); 15038 bp->fw_reset_timestamp = jiffies; 15039 return; 15040 } 15041 15042 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) { 15043 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) 15044 bnxt_fw_reset_writel(bp, i); 15045 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) { 15046 struct hwrm_fw_reset_input *req; 15047 15048 rc = hwrm_req_init(bp, req, HWRM_FW_RESET); 15049 if (!rc) { 15050 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG); 15051 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP; 15052 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP; 15053 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL; 15054 rc = hwrm_req_send(bp, req); 15055 } 15056 if (rc != -ENODEV) 15057 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc); 15058 } 15059 bp->fw_reset_timestamp = jiffies; 15060} 15061 15062static bool bnxt_fw_reset_timeout(struct bnxt *bp) 15063{ 15064 return time_after(jiffies, bp->fw_reset_timestamp + 15065 (bp->fw_reset_max_dsecs * HZ / 10)); 15066} 15067 15068static void bnxt_fw_reset_abort(struct bnxt *bp, int rc) 15069{ 15070 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 15071 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) 15072 bnxt_dl_health_fw_status_update(bp, false); 15073 bp->fw_reset_state = BNXT_FW_RESET_STATE_ABORT; 15074 netif_close(bp->dev); 15075} 15076 15077static void bnxt_fw_reset_task(struct work_struct *work) 15078{ 15079 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work); 15080 int rc = 0; 15081 15082 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 15083 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); 15084 return; 15085 } 15086 15087 switch (bp->fw_reset_state) { 15088 case BNXT_FW_RESET_STATE_POLL_VF: { 15089 int n = bnxt_get_registered_vfs(bp); 15090 int tmo; 15091 15092 if (n < 0) { 15093 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n", 15094 n, jiffies_to_msecs(jiffies - 15095 bp->fw_reset_timestamp)); 15096 goto fw_reset_abort; 15097 } else if (n > 0) { 15098 if (bnxt_fw_reset_timeout(bp)) { 15099 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 15100 bp->fw_reset_state = 0; 15101 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n", 15102 n); 15103 goto ulp_start; 15104 } 15105 bnxt_queue_fw_reset_work(bp, HZ / 10); 15106 return; 15107 } 15108 bp->fw_reset_timestamp = jiffies; 15109 netdev_lock(bp->dev); 15110 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) { 15111 bnxt_fw_reset_abort(bp, rc); 15112 netdev_unlock(bp->dev); 15113 goto ulp_start; 15114 } 15115 bnxt_fw_reset_close(bp); 15116 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { 15117 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; 15118 tmo = HZ / 10; 15119 } else { 15120 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 15121 tmo = bp->fw_reset_min_dsecs * HZ / 10; 15122 } 15123 netdev_unlock(bp->dev); 15124 bnxt_queue_fw_reset_work(bp, tmo); 15125 return; 15126 } 15127 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { 15128 u32 val; 15129 15130 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 15131 if (!(val & BNXT_FW_STATUS_SHUTDOWN) && 15132 !bnxt_fw_reset_timeout(bp)) { 15133 bnxt_queue_fw_reset_work(bp, HZ / 5); 15134 return; 15135 } 15136 15137 if (!bp->fw_health->primary) { 15138 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; 15139 15140 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 15141 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); 15142 return; 15143 } 15144 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; 15145 } 15146 fallthrough; 15147 case BNXT_FW_RESET_STATE_RESET_FW: 15148 bnxt_reset_all(bp); 15149 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; 15150 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); 15151 return; 15152 case BNXT_FW_RESET_STATE_ENABLE_DEV: 15153 bnxt_inv_fw_health_reg(bp); 15154 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && 15155 !bp->fw_reset_min_dsecs) { 15156 u16 val; 15157 15158 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val); 15159 if (val == 0xffff) { 15160 if (bnxt_fw_reset_timeout(bp)) { 15161 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); 15162 rc = -ETIMEDOUT; 15163 goto fw_reset_abort; 15164 } 15165 bnxt_queue_fw_reset_work(bp, HZ / 1000); 15166 return; 15167 } 15168 } 15169 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); 15170 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); 15171 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && 15172 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) 15173 bnxt_dl_remote_reload(bp); 15174 if (pci_enable_device(bp->pdev)) { 15175 netdev_err(bp->dev, "Cannot re-enable PCI device\n"); 15176 rc = -ENODEV; 15177 goto fw_reset_abort; 15178 } 15179 pci_set_master(bp->pdev); 15180 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; 15181 fallthrough; 15182 case BNXT_FW_RESET_STATE_POLL_FW: 15183 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT; 15184 rc = bnxt_hwrm_poll(bp); 15185 if (rc) { 15186 if (bnxt_fw_reset_timeout(bp)) { 15187 netdev_err(bp->dev, "Firmware reset aborted\n"); 15188 goto fw_reset_abort_status; 15189 } 15190 bnxt_queue_fw_reset_work(bp, HZ / 5); 15191 return; 15192 } 15193 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT; 15194 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; 15195 fallthrough; 15196 case BNXT_FW_RESET_STATE_OPENING: 15197 while (!netdev_trylock(bp->dev)) { 15198 bnxt_queue_fw_reset_work(bp, HZ / 10); 15199 return; 15200 } 15201 rc = bnxt_open(bp->dev); 15202 if (rc) { 15203 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); 15204 bnxt_fw_reset_abort(bp, rc); 15205 netdev_unlock(bp->dev); 15206 goto ulp_start; 15207 } 15208 15209 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && 15210 bp->fw_health->enabled) { 15211 bp->fw_health->last_fw_reset_cnt = 15212 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); 15213 } 15214 bp->fw_reset_state = 0; 15215 /* Make sure fw_reset_state is 0 before clearing the flag */ 15216 smp_mb__before_atomic(); 15217 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 15218 bnxt_ptp_reapply_pps(bp); 15219 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); 15220 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) { 15221 bnxt_dl_health_fw_recovery_done(bp); 15222 bnxt_dl_health_fw_status_update(bp, true); 15223 } 15224 netdev_unlock(bp->dev); 15225 bnxt_ulp_start(bp, 0); 15226 bnxt_reenable_sriov(bp); 15227 netdev_lock(bp->dev); 15228 bnxt_vf_reps_alloc(bp); 15229 bnxt_vf_reps_open(bp); 15230 netdev_unlock(bp->dev); 15231 break; 15232 } 15233 return; 15234 15235fw_reset_abort_status: 15236 if (bp->fw_health->status_reliable || 15237 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { 15238 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); 15239 15240 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts); 15241 } 15242fw_reset_abort: 15243 netdev_lock(bp->dev); 15244 bnxt_fw_reset_abort(bp, rc); 15245 netdev_unlock(bp->dev); 15246ulp_start: 15247 bnxt_ulp_start(bp, rc); 15248} 15249 15250static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev) 15251{ 15252 int rc; 15253 struct bnxt *bp = netdev_priv(dev); 15254 15255 SET_NETDEV_DEV(dev, &pdev->dev); 15256 15257 /* enable device (incl. PCI PM wakeup), and bus-mastering */ 15258 rc = pci_enable_device(pdev); 15259 if (rc) { 15260 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n"); 15261 goto init_err; 15262 } 15263 15264 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) { 15265 dev_err(&pdev->dev, 15266 "Cannot find PCI device base address, aborting\n"); 15267 rc = -ENODEV; 15268 goto init_err_disable; 15269 } 15270 15271 rc = pci_request_regions(pdev, DRV_MODULE_NAME); 15272 if (rc) { 15273 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n"); 15274 goto init_err_disable; 15275 } 15276 15277 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 && 15278 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) { 15279 dev_err(&pdev->dev, "System does not support DMA, aborting\n"); 15280 rc = -EIO; 15281 goto init_err_release; 15282 } 15283 15284 pci_set_master(pdev); 15285 15286 bp->dev = dev; 15287 bp->pdev = pdev; 15288 15289 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2() 15290 * determines the BAR size. 15291 */ 15292 bp->bar0 = pci_ioremap_bar(pdev, 0); 15293 if (!bp->bar0) { 15294 dev_err(&pdev->dev, "Cannot map device registers, aborting\n"); 15295 rc = -ENOMEM; 15296 goto init_err_release; 15297 } 15298 15299 bp->bar2 = pci_ioremap_bar(pdev, 4); 15300 if (!bp->bar2) { 15301 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n"); 15302 rc = -ENOMEM; 15303 goto init_err_release; 15304 } 15305 15306 INIT_WORK(&bp->sp_task, bnxt_sp_task); 15307 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task); 15308 15309 spin_lock_init(&bp->ntp_fltr_lock); 15310#if BITS_PER_LONG == 32 15311 spin_lock_init(&bp->db_lock); 15312#endif 15313 15314 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE; 15315 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE; 15316 15317 timer_setup(&bp->timer, bnxt_timer, 0); 15318 bp->current_interval = BNXT_TIMER_INTERVAL; 15319 15320 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID; 15321 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID; 15322 15323 clear_bit(BNXT_STATE_OPEN, &bp->state); 15324 return 0; 15325 15326init_err_release: 15327 bnxt_unmap_bars(bp, pdev); 15328 pci_release_regions(pdev); 15329 15330init_err_disable: 15331 pci_disable_device(pdev); 15332 15333init_err: 15334 return rc; 15335} 15336 15337static int bnxt_change_mac_addr(struct net_device *dev, void *p) 15338{ 15339 struct sockaddr *addr = p; 15340 struct bnxt *bp = netdev_priv(dev); 15341 int rc = 0; 15342 15343 netdev_assert_locked(dev); 15344 15345 if (!is_valid_ether_addr(addr->sa_data)) 15346 return -EADDRNOTAVAIL; 15347 15348 if (ether_addr_equal(addr->sa_data, dev->dev_addr)) 15349 return 0; 15350 15351 rc = bnxt_approve_mac(bp, addr->sa_data, true); 15352 if (rc) 15353 return rc; 15354 15355 eth_hw_addr_set(dev, addr->sa_data); 15356 bnxt_clear_usr_fltrs(bp, true); 15357 if (netif_running(dev)) { 15358 bnxt_close_nic(bp, false, false); 15359 rc = bnxt_open_nic(bp, false, false); 15360 } 15361 15362 return rc; 15363} 15364 15365static int bnxt_change_mtu(struct net_device *dev, int new_mtu) 15366{ 15367 struct bnxt *bp = netdev_priv(dev); 15368 15369 netdev_assert_locked(dev); 15370 15371 if (netif_running(dev)) 15372 bnxt_close_nic(bp, true, false); 15373 15374 WRITE_ONCE(dev->mtu, new_mtu); 15375 15376 /* MTU change may change the AGG ring settings if an XDP multi-buffer 15377 * program is attached. We need to set the AGG rings settings and 15378 * rx_skb_func accordingly. 15379 */ 15380 if (READ_ONCE(bp->xdp_prog)) 15381 bnxt_set_rx_skb_mode(bp, true); 15382 15383 bnxt_set_ring_params(bp); 15384 15385 if (netif_running(dev)) 15386 return bnxt_open_nic(bp, true, false); 15387 15388 return 0; 15389} 15390 15391void bnxt_set_cp_rings(struct bnxt *bp, bool sh) 15392{ 15393 int tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); 15394 15395 bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) : 15396 tx_cp + bp->rx_nr_rings; 15397} 15398 15399int bnxt_setup_mq_tc(struct net_device *dev, u8 tc) 15400{ 15401 struct bnxt *bp = netdev_priv(dev); 15402 bool sh = false; 15403 int rc; 15404 15405 if (tc > bp->max_tc) { 15406 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n", 15407 tc, bp->max_tc); 15408 return -EINVAL; 15409 } 15410 15411 if (bp->num_tc == tc) 15412 return 0; 15413 15414 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 15415 sh = true; 15416 15417 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 15418 sh, tc, bp->tx_nr_rings_xdp); 15419 if (rc) 15420 return rc; 15421 15422 /* Needs to close the device and do hw resource re-allocations */ 15423 if (netif_running(bp->dev)) 15424 bnxt_close_nic(bp, true, false); 15425 15426 if (tc) { 15427 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc; 15428 netdev_set_num_tc(dev, tc); 15429 bp->num_tc = tc; 15430 } else { 15431 bp->tx_nr_rings = bp->tx_nr_rings_per_tc; 15432 netdev_reset_tc(dev); 15433 bp->num_tc = 0; 15434 } 15435 bp->tx_nr_rings += bp->tx_nr_rings_xdp; 15436 bnxt_set_cp_rings(bp, sh); 15437 15438 if (netif_running(bp->dev)) 15439 return bnxt_open_nic(bp, true, false); 15440 15441 return 0; 15442} 15443 15444static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 15445 void *cb_priv) 15446{ 15447 struct bnxt *bp = cb_priv; 15448 15449 if (!bnxt_tc_flower_enabled(bp) || 15450 !tc_cls_can_offload_and_chain0(bp->dev, type_data)) 15451 return -EOPNOTSUPP; 15452 15453 switch (type) { 15454 case TC_SETUP_CLSFLOWER: 15455 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data); 15456 default: 15457 return -EOPNOTSUPP; 15458 } 15459} 15460 15461LIST_HEAD(bnxt_block_cb_list); 15462 15463static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, 15464 void *type_data) 15465{ 15466 struct bnxt *bp = netdev_priv(dev); 15467 15468 switch (type) { 15469 case TC_SETUP_BLOCK: 15470 return flow_block_cb_setup_simple(type_data, 15471 &bnxt_block_cb_list, 15472 bnxt_setup_tc_block_cb, 15473 bp, bp, true); 15474 case TC_SETUP_QDISC_MQPRIO: { 15475 struct tc_mqprio_qopt *mqprio = type_data; 15476 15477 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 15478 15479 return bnxt_setup_mq_tc(dev, mqprio->num_tc); 15480 } 15481 default: 15482 return -EOPNOTSUPP; 15483 } 15484} 15485 15486u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys, 15487 const struct sk_buff *skb) 15488{ 15489 struct bnxt_vnic_info *vnic; 15490 15491 if (skb) 15492 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK; 15493 15494 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT]; 15495 return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key); 15496} 15497 15498int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr, 15499 u32 idx) 15500{ 15501 struct hlist_head *head; 15502 int bit_id; 15503 15504 spin_lock_bh(&bp->ntp_fltr_lock); 15505 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, bp->max_fltr, 0); 15506 if (bit_id < 0) { 15507 spin_unlock_bh(&bp->ntp_fltr_lock); 15508 return -ENOMEM; 15509 } 15510 15511 fltr->base.sw_id = (u16)bit_id; 15512 fltr->base.type = BNXT_FLTR_TYPE_NTUPLE; 15513 fltr->base.flags |= BNXT_ACT_RING_DST; 15514 head = &bp->ntp_fltr_hash_tbl[idx]; 15515 hlist_add_head_rcu(&fltr->base.hash, head); 15516 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state); 15517 bnxt_insert_usr_fltr(bp, &fltr->base); 15518 bp->ntp_fltr_count++; 15519 spin_unlock_bh(&bp->ntp_fltr_lock); 15520 return 0; 15521} 15522 15523static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1, 15524 struct bnxt_ntuple_filter *f2) 15525{ 15526 struct bnxt_flow_masks *masks1 = &f1->fmasks; 15527 struct bnxt_flow_masks *masks2 = &f2->fmasks; 15528 struct flow_keys *keys1 = &f1->fkeys; 15529 struct flow_keys *keys2 = &f2->fkeys; 15530 15531 if (keys1->basic.n_proto != keys2->basic.n_proto || 15532 keys1->basic.ip_proto != keys2->basic.ip_proto) 15533 return false; 15534 15535 if (keys1->basic.n_proto == htons(ETH_P_IP)) { 15536 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src || 15537 masks1->addrs.v4addrs.src != masks2->addrs.v4addrs.src || 15538 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst || 15539 masks1->addrs.v4addrs.dst != masks2->addrs.v4addrs.dst) 15540 return false; 15541 } else { 15542 if (!ipv6_addr_equal(&keys1->addrs.v6addrs.src, 15543 &keys2->addrs.v6addrs.src) || 15544 !ipv6_addr_equal(&masks1->addrs.v6addrs.src, 15545 &masks2->addrs.v6addrs.src) || 15546 !ipv6_addr_equal(&keys1->addrs.v6addrs.dst, 15547 &keys2->addrs.v6addrs.dst) || 15548 !ipv6_addr_equal(&masks1->addrs.v6addrs.dst, 15549 &masks2->addrs.v6addrs.dst)) 15550 return false; 15551 } 15552 15553 return keys1->ports.src == keys2->ports.src && 15554 masks1->ports.src == masks2->ports.src && 15555 keys1->ports.dst == keys2->ports.dst && 15556 masks1->ports.dst == masks2->ports.dst && 15557 keys1->control.flags == keys2->control.flags && 15558 f1->l2_fltr == f2->l2_fltr; 15559} 15560 15561struct bnxt_ntuple_filter * 15562bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp, 15563 struct bnxt_ntuple_filter *fltr, u32 idx) 15564{ 15565 struct bnxt_ntuple_filter *f; 15566 struct hlist_head *head; 15567 15568 head = &bp->ntp_fltr_hash_tbl[idx]; 15569 hlist_for_each_entry_rcu(f, head, base.hash) { 15570 if (bnxt_fltr_match(f, fltr)) 15571 return f; 15572 } 15573 return NULL; 15574} 15575 15576#ifdef CONFIG_RFS_ACCEL 15577static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 15578 u16 rxq_index, u32 flow_id) 15579{ 15580 struct bnxt *bp = netdev_priv(dev); 15581 struct bnxt_ntuple_filter *fltr, *new_fltr; 15582 struct flow_keys *fkeys; 15583 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb); 15584 struct bnxt_l2_filter *l2_fltr; 15585 int rc = 0, idx; 15586 u32 flags; 15587 15588 if (ether_addr_equal(dev->dev_addr, eth->h_dest)) { 15589 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0]; 15590 atomic_inc(&l2_fltr->refcnt); 15591 } else { 15592 struct bnxt_l2_key key; 15593 15594 ether_addr_copy(key.dst_mac_addr, eth->h_dest); 15595 key.vlan = 0; 15596 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key); 15597 if (!l2_fltr) 15598 return -EINVAL; 15599 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) { 15600 bnxt_del_l2_filter(bp, l2_fltr); 15601 return -EINVAL; 15602 } 15603 } 15604 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC); 15605 if (!new_fltr) { 15606 bnxt_del_l2_filter(bp, l2_fltr); 15607 return -ENOMEM; 15608 } 15609 15610 fkeys = &new_fltr->fkeys; 15611 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) { 15612 rc = -EPROTONOSUPPORT; 15613 goto err_free; 15614 } 15615 15616 if ((fkeys->basic.n_proto != htons(ETH_P_IP) && 15617 fkeys->basic.n_proto != htons(ETH_P_IPV6)) || 15618 ((fkeys->basic.ip_proto != IPPROTO_TCP) && 15619 (fkeys->basic.ip_proto != IPPROTO_UDP))) { 15620 rc = -EPROTONOSUPPORT; 15621 goto err_free; 15622 } 15623 new_fltr->fmasks = BNXT_FLOW_IPV4_MASK_ALL; 15624 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) { 15625 if (bp->hwrm_spec_code < 0x10601) { 15626 rc = -EPROTONOSUPPORT; 15627 goto err_free; 15628 } 15629 new_fltr->fmasks = BNXT_FLOW_IPV6_MASK_ALL; 15630 } 15631 flags = fkeys->control.flags; 15632 if (((flags & FLOW_DIS_ENCAPSULATION) && 15633 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) { 15634 rc = -EPROTONOSUPPORT; 15635 goto err_free; 15636 } 15637 new_fltr->l2_fltr = l2_fltr; 15638 15639 idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb); 15640 rcu_read_lock(); 15641 fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx); 15642 if (fltr) { 15643 rc = fltr->base.sw_id; 15644 rcu_read_unlock(); 15645 goto err_free; 15646 } 15647 rcu_read_unlock(); 15648 15649 new_fltr->flow_id = flow_id; 15650 new_fltr->base.rxq = rxq_index; 15651 rc = bnxt_insert_ntp_filter(bp, new_fltr, idx); 15652 if (!rc) { 15653 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT); 15654 return new_fltr->base.sw_id; 15655 } 15656 15657err_free: 15658 bnxt_del_l2_filter(bp, l2_fltr); 15659 kfree(new_fltr); 15660 return rc; 15661} 15662#endif 15663 15664void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr) 15665{ 15666 spin_lock_bh(&bp->ntp_fltr_lock); 15667 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) { 15668 spin_unlock_bh(&bp->ntp_fltr_lock); 15669 return; 15670 } 15671 hlist_del_rcu(&fltr->base.hash); 15672 bnxt_del_one_usr_fltr(bp, &fltr->base); 15673 bp->ntp_fltr_count--; 15674 spin_unlock_bh(&bp->ntp_fltr_lock); 15675 bnxt_del_l2_filter(bp, fltr->l2_fltr); 15676 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap); 15677 kfree_rcu(fltr, base.rcu); 15678} 15679 15680static void bnxt_cfg_ntp_filters(struct bnxt *bp) 15681{ 15682#ifdef CONFIG_RFS_ACCEL 15683 int i; 15684 15685 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) { 15686 struct hlist_head *head; 15687 struct hlist_node *tmp; 15688 struct bnxt_ntuple_filter *fltr; 15689 int rc; 15690 15691 head = &bp->ntp_fltr_hash_tbl[i]; 15692 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) { 15693 bool del = false; 15694 15695 if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) { 15696 if (fltr->base.flags & BNXT_ACT_NO_AGING) 15697 continue; 15698 if (rps_may_expire_flow(bp->dev, fltr->base.rxq, 15699 fltr->flow_id, 15700 fltr->base.sw_id)) { 15701 bnxt_hwrm_cfa_ntuple_filter_free(bp, 15702 fltr); 15703 del = true; 15704 } 15705 } else { 15706 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp, 15707 fltr); 15708 if (rc) 15709 del = true; 15710 else 15711 set_bit(BNXT_FLTR_VALID, &fltr->base.state); 15712 } 15713 15714 if (del) 15715 bnxt_del_ntp_filter(bp, fltr); 15716 } 15717 } 15718#endif 15719} 15720 15721static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, 15722 unsigned int entry, struct udp_tunnel_info *ti) 15723{ 15724 struct bnxt *bp = netdev_priv(netdev); 15725 unsigned int cmd; 15726 15727 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 15728 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN; 15729 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE) 15730 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE; 15731 else 15732 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE; 15733 15734 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd); 15735} 15736 15737static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, 15738 unsigned int entry, struct udp_tunnel_info *ti) 15739{ 15740 struct bnxt *bp = netdev_priv(netdev); 15741 unsigned int cmd; 15742 15743 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 15744 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN; 15745 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE) 15746 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE; 15747 else 15748 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE; 15749 15750 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd); 15751} 15752 15753static const struct udp_tunnel_nic_info bnxt_udp_tunnels = { 15754 .set_port = bnxt_udp_tunnel_set_port, 15755 .unset_port = bnxt_udp_tunnel_unset_port, 15756 .flags = UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 15757 .tables = { 15758 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 15759 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 15760 }, 15761}, bnxt_udp_tunnels_p7 = { 15762 .set_port = bnxt_udp_tunnel_set_port, 15763 .unset_port = bnxt_udp_tunnel_unset_port, 15764 .flags = UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 15765 .tables = { 15766 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 15767 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, }, 15768 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, }, 15769 }, 15770}; 15771 15772static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 15773 struct net_device *dev, u32 filter_mask, 15774 int nlflags) 15775{ 15776 struct bnxt *bp = netdev_priv(dev); 15777 15778 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0, 15779 nlflags, filter_mask, NULL); 15780} 15781 15782static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 15783 u16 flags, struct netlink_ext_ack *extack) 15784{ 15785 struct bnxt *bp = netdev_priv(dev); 15786 struct nlattr *attr, *br_spec; 15787 int rem, rc = 0; 15788 15789 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp)) 15790 return -EOPNOTSUPP; 15791 15792 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 15793 if (!br_spec) 15794 return -EINVAL; 15795 15796 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 15797 u16 mode; 15798 15799 mode = nla_get_u16(attr); 15800 if (mode == bp->br_mode) 15801 break; 15802 15803 rc = bnxt_hwrm_set_br_mode(bp, mode); 15804 if (!rc) 15805 bp->br_mode = mode; 15806 break; 15807 } 15808 return rc; 15809} 15810 15811int bnxt_get_port_parent_id(struct net_device *dev, 15812 struct netdev_phys_item_id *ppid) 15813{ 15814 struct bnxt *bp = netdev_priv(dev); 15815 15816 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV) 15817 return -EOPNOTSUPP; 15818 15819 /* The PF and it's VF-reps only support the switchdev framework */ 15820 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID)) 15821 return -EOPNOTSUPP; 15822 15823 ppid->id_len = sizeof(bp->dsn); 15824 memcpy(ppid->id, bp->dsn, ppid->id_len); 15825 15826 return 0; 15827} 15828 15829static const struct net_device_ops bnxt_netdev_ops = { 15830 .ndo_open = bnxt_open, 15831 .ndo_start_xmit = bnxt_start_xmit, 15832 .ndo_stop = bnxt_close, 15833 .ndo_get_stats64 = bnxt_get_stats64, 15834 .ndo_set_rx_mode = bnxt_set_rx_mode, 15835 .ndo_eth_ioctl = bnxt_ioctl, 15836 .ndo_validate_addr = eth_validate_addr, 15837 .ndo_set_mac_address = bnxt_change_mac_addr, 15838 .ndo_change_mtu = bnxt_change_mtu, 15839 .ndo_fix_features = bnxt_fix_features, 15840 .ndo_set_features = bnxt_set_features, 15841 .ndo_features_check = bnxt_features_check, 15842 .ndo_tx_timeout = bnxt_tx_timeout, 15843#ifdef CONFIG_BNXT_SRIOV 15844 .ndo_get_vf_config = bnxt_get_vf_config, 15845 .ndo_set_vf_mac = bnxt_set_vf_mac, 15846 .ndo_set_vf_vlan = bnxt_set_vf_vlan, 15847 .ndo_set_vf_rate = bnxt_set_vf_bw, 15848 .ndo_set_vf_link_state = bnxt_set_vf_link_state, 15849 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk, 15850 .ndo_set_vf_trust = bnxt_set_vf_trust, 15851#endif 15852 .ndo_setup_tc = bnxt_setup_tc, 15853#ifdef CONFIG_RFS_ACCEL 15854 .ndo_rx_flow_steer = bnxt_rx_flow_steer, 15855#endif 15856 .ndo_bpf = bnxt_xdp, 15857 .ndo_xdp_xmit = bnxt_xdp_xmit, 15858 .ndo_bridge_getlink = bnxt_bridge_getlink, 15859 .ndo_bridge_setlink = bnxt_bridge_setlink, 15860 .ndo_hwtstamp_get = bnxt_hwtstamp_get, 15861 .ndo_hwtstamp_set = bnxt_hwtstamp_set, 15862}; 15863 15864static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, 15865 struct netdev_queue_stats_rx *stats) 15866{ 15867 struct bnxt *bp = netdev_priv(dev); 15868 struct bnxt_cp_ring_info *cpr; 15869 u64 *sw; 15870 15871 if (!bp->bnapi) 15872 return; 15873 15874 cpr = &bp->bnapi[i]->cp_ring; 15875 sw = cpr->stats.sw_stats; 15876 15877 stats->packets = 0; 15878 stats->packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts); 15879 stats->packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts); 15880 stats->packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts); 15881 15882 stats->bytes = 0; 15883 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes); 15884 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes); 15885 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes); 15886 15887 stats->alloc_fail = cpr->sw_stats->rx.rx_oom_discards; 15888} 15889 15890static void bnxt_get_queue_stats_tx(struct net_device *dev, int i, 15891 struct netdev_queue_stats_tx *stats) 15892{ 15893 struct bnxt *bp = netdev_priv(dev); 15894 struct bnxt_napi *bnapi; 15895 u64 *sw; 15896 15897 if (!bp->tx_ring) 15898 return; 15899 15900 bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi; 15901 sw = bnapi->cp_ring.stats.sw_stats; 15902 15903 stats->packets = 0; 15904 stats->packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts); 15905 stats->packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts); 15906 stats->packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts); 15907 15908 stats->bytes = 0; 15909 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes); 15910 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes); 15911 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes); 15912} 15913 15914static void bnxt_get_base_stats(struct net_device *dev, 15915 struct netdev_queue_stats_rx *rx, 15916 struct netdev_queue_stats_tx *tx) 15917{ 15918 struct bnxt *bp = netdev_priv(dev); 15919 15920 rx->packets = bp->net_stats_prev.rx_packets; 15921 rx->bytes = bp->net_stats_prev.rx_bytes; 15922 rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards; 15923 15924 tx->packets = bp->net_stats_prev.tx_packets; 15925 tx->bytes = bp->net_stats_prev.tx_bytes; 15926} 15927 15928static const struct netdev_stat_ops bnxt_stat_ops = { 15929 .get_queue_stats_rx = bnxt_get_queue_stats_rx, 15930 .get_queue_stats_tx = bnxt_get_queue_stats_tx, 15931 .get_base_stats = bnxt_get_base_stats, 15932}; 15933 15934static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) 15935{ 15936 struct bnxt_rx_ring_info *rxr, *clone; 15937 struct bnxt *bp = netdev_priv(dev); 15938 struct bnxt_ring_struct *ring; 15939 int rc; 15940 15941 if (!bp->rx_ring) 15942 return -ENETDOWN; 15943 15944 rxr = &bp->rx_ring[idx]; 15945 clone = qmem; 15946 memcpy(clone, rxr, sizeof(*rxr)); 15947 bnxt_init_rx_ring_struct(bp, clone); 15948 bnxt_reset_rx_ring_struct(bp, clone); 15949 15950 clone->rx_prod = 0; 15951 clone->rx_agg_prod = 0; 15952 clone->rx_sw_agg_prod = 0; 15953 clone->rx_next_cons = 0; 15954 clone->need_head_pool = false; 15955 15956 rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); 15957 if (rc) 15958 return rc; 15959 15960 rc = xdp_rxq_info_reg(&clone->xdp_rxq, bp->dev, idx, 0); 15961 if (rc < 0) 15962 goto err_page_pool_destroy; 15963 15964 rc = xdp_rxq_info_reg_mem_model(&clone->xdp_rxq, 15965 MEM_TYPE_PAGE_POOL, 15966 clone->page_pool); 15967 if (rc) 15968 goto err_rxq_info_unreg; 15969 15970 ring = &clone->rx_ring_struct; 15971 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 15972 if (rc) 15973 goto err_free_rx_ring; 15974 15975 if (bp->flags & BNXT_FLAG_AGG_RINGS) { 15976 ring = &clone->rx_agg_ring_struct; 15977 rc = bnxt_alloc_ring(bp, &ring->ring_mem); 15978 if (rc) 15979 goto err_free_rx_agg_ring; 15980 15981 rc = bnxt_alloc_rx_agg_bmap(bp, clone); 15982 if (rc) 15983 goto err_free_rx_agg_ring; 15984 } 15985 15986 if (bp->flags & BNXT_FLAG_TPA) { 15987 rc = bnxt_alloc_one_tpa_info(bp, clone); 15988 if (rc) 15989 goto err_free_tpa_info; 15990 } 15991 15992 bnxt_init_one_rx_ring_rxbd(bp, clone); 15993 bnxt_init_one_rx_agg_ring_rxbd(bp, clone); 15994 15995 bnxt_alloc_one_rx_ring_skb(bp, clone, idx); 15996 if (bp->flags & BNXT_FLAG_AGG_RINGS) 15997 bnxt_alloc_one_rx_ring_netmem(bp, clone, idx); 15998 if (bp->flags & BNXT_FLAG_TPA) 15999 bnxt_alloc_one_tpa_info_data(bp, clone); 16000 16001 return 0; 16002 16003err_free_tpa_info: 16004 bnxt_free_one_tpa_info(bp, clone); 16005err_free_rx_agg_ring: 16006 bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem); 16007err_free_rx_ring: 16008 bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem); 16009err_rxq_info_unreg: 16010 xdp_rxq_info_unreg(&clone->xdp_rxq); 16011err_page_pool_destroy: 16012 page_pool_destroy(clone->page_pool); 16013 page_pool_destroy(clone->head_pool); 16014 clone->page_pool = NULL; 16015 clone->head_pool = NULL; 16016 return rc; 16017} 16018 16019static void bnxt_queue_mem_free(struct net_device *dev, void *qmem) 16020{ 16021 struct bnxt_rx_ring_info *rxr = qmem; 16022 struct bnxt *bp = netdev_priv(dev); 16023 struct bnxt_ring_struct *ring; 16024 16025 bnxt_free_one_rx_ring_skbs(bp, rxr); 16026 bnxt_free_one_tpa_info(bp, rxr); 16027 16028 xdp_rxq_info_unreg(&rxr->xdp_rxq); 16029 16030 page_pool_destroy(rxr->page_pool); 16031 page_pool_destroy(rxr->head_pool); 16032 rxr->page_pool = NULL; 16033 rxr->head_pool = NULL; 16034 16035 ring = &rxr->rx_ring_struct; 16036 bnxt_free_ring(bp, &ring->ring_mem); 16037 16038 ring = &rxr->rx_agg_ring_struct; 16039 bnxt_free_ring(bp, &ring->ring_mem); 16040 16041 kfree(rxr->rx_agg_bmap); 16042 rxr->rx_agg_bmap = NULL; 16043} 16044 16045static void bnxt_copy_rx_ring(struct bnxt *bp, 16046 struct bnxt_rx_ring_info *dst, 16047 struct bnxt_rx_ring_info *src) 16048{ 16049 struct bnxt_ring_mem_info *dst_rmem, *src_rmem; 16050 struct bnxt_ring_struct *dst_ring, *src_ring; 16051 int i; 16052 16053 dst_ring = &dst->rx_ring_struct; 16054 dst_rmem = &dst_ring->ring_mem; 16055 src_ring = &src->rx_ring_struct; 16056 src_rmem = &src_ring->ring_mem; 16057 16058 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); 16059 WARN_ON(dst_rmem->page_size != src_rmem->page_size); 16060 WARN_ON(dst_rmem->flags != src_rmem->flags); 16061 WARN_ON(dst_rmem->depth != src_rmem->depth); 16062 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size); 16063 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem); 16064 16065 dst_rmem->pg_tbl = src_rmem->pg_tbl; 16066 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map; 16067 *dst_rmem->vmem = *src_rmem->vmem; 16068 for (i = 0; i < dst_rmem->nr_pages; i++) { 16069 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i]; 16070 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i]; 16071 } 16072 16073 if (!(bp->flags & BNXT_FLAG_AGG_RINGS)) 16074 return; 16075 16076 dst_ring = &dst->rx_agg_ring_struct; 16077 dst_rmem = &dst_ring->ring_mem; 16078 src_ring = &src->rx_agg_ring_struct; 16079 src_rmem = &src_ring->ring_mem; 16080 16081 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); 16082 WARN_ON(dst_rmem->page_size != src_rmem->page_size); 16083 WARN_ON(dst_rmem->flags != src_rmem->flags); 16084 WARN_ON(dst_rmem->depth != src_rmem->depth); 16085 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size); 16086 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem); 16087 WARN_ON(dst->rx_agg_bmap_size != src->rx_agg_bmap_size); 16088 16089 dst_rmem->pg_tbl = src_rmem->pg_tbl; 16090 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map; 16091 *dst_rmem->vmem = *src_rmem->vmem; 16092 for (i = 0; i < dst_rmem->nr_pages; i++) { 16093 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i]; 16094 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i]; 16095 } 16096 16097 dst->rx_agg_bmap = src->rx_agg_bmap; 16098} 16099 16100static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) 16101{ 16102 struct bnxt *bp = netdev_priv(dev); 16103 struct bnxt_rx_ring_info *rxr, *clone; 16104 struct bnxt_cp_ring_info *cpr; 16105 struct bnxt_vnic_info *vnic; 16106 struct bnxt_napi *bnapi; 16107 int i, rc; 16108 u16 mru; 16109 16110 rxr = &bp->rx_ring[idx]; 16111 clone = qmem; 16112 16113 rxr->rx_prod = clone->rx_prod; 16114 rxr->rx_agg_prod = clone->rx_agg_prod; 16115 rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod; 16116 rxr->rx_next_cons = clone->rx_next_cons; 16117 rxr->rx_tpa = clone->rx_tpa; 16118 rxr->rx_tpa_idx_map = clone->rx_tpa_idx_map; 16119 rxr->page_pool = clone->page_pool; 16120 rxr->head_pool = clone->head_pool; 16121 rxr->xdp_rxq = clone->xdp_rxq; 16122 rxr->need_head_pool = clone->need_head_pool; 16123 16124 bnxt_copy_rx_ring(bp, rxr, clone); 16125 16126 bnapi = rxr->bnapi; 16127 cpr = &bnapi->cp_ring; 16128 16129 /* All rings have been reserved and previously allocated. 16130 * Reallocating with the same parameters should never fail. 16131 */ 16132 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr); 16133 if (rc) 16134 goto err_reset; 16135 16136 if (bp->tph_mode) { 16137 rc = bnxt_hwrm_cp_ring_alloc_p5(bp, rxr->rx_cpr); 16138 if (rc) 16139 goto err_reset; 16140 } 16141 16142 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr); 16143 if (rc) 16144 goto err_reset; 16145 16146 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod); 16147 if (bp->flags & BNXT_FLAG_AGG_RINGS) 16148 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod); 16149 16150 if (bp->flags & BNXT_FLAG_SHARED_RINGS) { 16151 rc = bnxt_tx_queue_start(bp, idx); 16152 if (rc) 16153 goto err_reset; 16154 } 16155 16156 bnxt_enable_rx_page_pool(rxr); 16157 napi_enable_locked(&bnapi->napi); 16158 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 16159 16160 mru = bp->dev->mtu + VLAN_ETH_HLEN; 16161 for (i = 0; i < bp->nr_vnics; i++) { 16162 vnic = &bp->vnic_info[i]; 16163 16164 rc = bnxt_set_vnic_mru_p5(bp, vnic, mru, idx); 16165 if (rc) 16166 return rc; 16167 } 16168 return bnxt_set_rss_ctx_vnic_mru(bp, mru, idx); 16169 16170err_reset: 16171 netdev_err(bp->dev, "Unexpected HWRM error during queue start rc: %d\n", 16172 rc); 16173 napi_enable_locked(&bnapi->napi); 16174 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); 16175 bnxt_reset_task(bp, true); 16176 return rc; 16177} 16178 16179static int bnxt_queue_stop(struct net_device *dev, void *qmem, int idx) 16180{ 16181 struct bnxt *bp = netdev_priv(dev); 16182 struct bnxt_rx_ring_info *rxr; 16183 struct bnxt_cp_ring_info *cpr; 16184 struct bnxt_vnic_info *vnic; 16185 struct bnxt_napi *bnapi; 16186 int i; 16187 16188 for (i = 0; i < bp->nr_vnics; i++) { 16189 vnic = &bp->vnic_info[i]; 16190 16191 bnxt_set_vnic_mru_p5(bp, vnic, 0, idx); 16192 } 16193 bnxt_set_rss_ctx_vnic_mru(bp, 0, idx); 16194 /* Make sure NAPI sees that the VNIC is disabled */ 16195 synchronize_net(); 16196 rxr = &bp->rx_ring[idx]; 16197 bnapi = rxr->bnapi; 16198 cpr = &bnapi->cp_ring; 16199 cancel_work_sync(&cpr->dim.work); 16200 bnxt_hwrm_rx_ring_free(bp, rxr, false); 16201 bnxt_hwrm_rx_agg_ring_free(bp, rxr, false); 16202 page_pool_disable_direct_recycling(rxr->page_pool); 16203 if (bnxt_separate_head_pool(rxr)) 16204 page_pool_disable_direct_recycling(rxr->head_pool); 16205 16206 if (bp->flags & BNXT_FLAG_SHARED_RINGS) 16207 bnxt_tx_queue_stop(bp, idx); 16208 16209 /* Disable NAPI now after freeing the rings because HWRM_RING_FREE 16210 * completion is handled in NAPI to guarantee no more DMA on that ring 16211 * after seeing the completion. 16212 */ 16213 napi_disable_locked(&bnapi->napi); 16214 16215 if (bp->tph_mode) { 16216 bnxt_hwrm_cp_ring_free(bp, rxr->rx_cpr); 16217 bnxt_clear_one_cp_ring(bp, rxr->rx_cpr); 16218 } 16219 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons); 16220 16221 memcpy(qmem, rxr, sizeof(*rxr)); 16222 bnxt_init_rx_ring_struct(bp, qmem); 16223 16224 return 0; 16225} 16226 16227static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = { 16228 .ndo_queue_mem_size = sizeof(struct bnxt_rx_ring_info), 16229 .ndo_queue_mem_alloc = bnxt_queue_mem_alloc, 16230 .ndo_queue_mem_free = bnxt_queue_mem_free, 16231 .ndo_queue_start = bnxt_queue_start, 16232 .ndo_queue_stop = bnxt_queue_stop, 16233}; 16234 16235static void bnxt_remove_one(struct pci_dev *pdev) 16236{ 16237 struct net_device *dev = pci_get_drvdata(pdev); 16238 struct bnxt *bp = netdev_priv(dev); 16239 16240 if (BNXT_PF(bp)) 16241 __bnxt_sriov_disable(bp); 16242 16243 bnxt_rdma_aux_device_del(bp); 16244 16245 unregister_netdev(dev); 16246 bnxt_ptp_clear(bp); 16247 16248 bnxt_rdma_aux_device_uninit(bp); 16249 16250 bnxt_free_l2_filters(bp, true); 16251 bnxt_free_ntp_fltrs(bp, true); 16252 WARN_ON(bp->num_rss_ctx); 16253 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 16254 /* Flush any pending tasks */ 16255 cancel_work_sync(&bp->sp_task); 16256 cancel_delayed_work_sync(&bp->fw_reset_task); 16257 bp->sp_event = 0; 16258 16259 bnxt_dl_fw_reporters_destroy(bp); 16260 bnxt_dl_unregister(bp); 16261 bnxt_shutdown_tc(bp); 16262 16263 bnxt_clear_int_mode(bp); 16264 bnxt_hwrm_func_drv_unrgtr(bp); 16265 bnxt_free_hwrm_resources(bp); 16266 bnxt_hwmon_uninit(bp); 16267 bnxt_ethtool_free(bp); 16268 bnxt_dcb_free(bp); 16269 kfree(bp->ptp_cfg); 16270 bp->ptp_cfg = NULL; 16271 kfree(bp->fw_health); 16272 bp->fw_health = NULL; 16273 bnxt_cleanup_pci(bp); 16274 bnxt_free_ctx_mem(bp, true); 16275 bnxt_free_crash_dump_mem(bp); 16276 kfree(bp->rss_indir_tbl); 16277 bp->rss_indir_tbl = NULL; 16278 bnxt_free_port_stats(bp); 16279 free_netdev(dev); 16280} 16281 16282static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt) 16283{ 16284 int rc = 0; 16285 struct bnxt_link_info *link_info = &bp->link_info; 16286 16287 bp->phy_flags = 0; 16288 rc = bnxt_hwrm_phy_qcaps(bp); 16289 if (rc) { 16290 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n", 16291 rc); 16292 return rc; 16293 } 16294 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS) 16295 bp->dev->priv_flags |= IFF_SUPP_NOFCS; 16296 else 16297 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS; 16298 16299 bp->mac_flags = 0; 16300 bnxt_hwrm_mac_qcaps(bp); 16301 16302 if (!fw_dflt) 16303 return 0; 16304 16305 mutex_lock(&bp->link_lock); 16306 rc = bnxt_update_link(bp, false); 16307 if (rc) { 16308 mutex_unlock(&bp->link_lock); 16309 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n", 16310 rc); 16311 return rc; 16312 } 16313 16314 /* Older firmware does not have supported_auto_speeds, so assume 16315 * that all supported speeds can be autonegotiated. 16316 */ 16317 if (link_info->auto_link_speeds && !link_info->support_auto_speeds) 16318 link_info->support_auto_speeds = link_info->support_speeds; 16319 16320 bnxt_init_ethtool_link_settings(bp); 16321 mutex_unlock(&bp->link_lock); 16322 return 0; 16323} 16324 16325static int bnxt_get_max_irq(struct pci_dev *pdev) 16326{ 16327 u16 ctrl; 16328 16329 if (!pdev->msix_cap) 16330 return 1; 16331 16332 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 16333 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1; 16334} 16335 16336static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, 16337 int *max_cp) 16338{ 16339 struct bnxt_hw_resc *hw_resc = &bp->hw_resc; 16340 int max_ring_grps = 0, max_irq; 16341 16342 *max_tx = hw_resc->max_tx_rings; 16343 *max_rx = hw_resc->max_rx_rings; 16344 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp); 16345 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) - 16346 bnxt_get_ulp_msix_num_in_use(bp), 16347 hw_resc->max_stat_ctxs - 16348 bnxt_get_ulp_stat_ctxs_in_use(bp)); 16349 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) 16350 *max_cp = min_t(int, *max_cp, max_irq); 16351 max_ring_grps = hw_resc->max_hw_ring_grps; 16352 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) { 16353 *max_cp -= 1; 16354 *max_rx -= 2; 16355 } 16356 if (bp->flags & BNXT_FLAG_AGG_RINGS) 16357 *max_rx >>= 1; 16358 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { 16359 int rc; 16360 16361 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false); 16362 if (rc) { 16363 *max_rx = 0; 16364 *max_tx = 0; 16365 } 16366 /* On P5 chips, max_cp output param should be available NQs */ 16367 *max_cp = max_irq; 16368 } 16369 *max_rx = min_t(int, *max_rx, max_ring_grps); 16370} 16371 16372int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared) 16373{ 16374 int rx, tx, cp; 16375 16376 _bnxt_get_max_rings(bp, &rx, &tx, &cp); 16377 *max_rx = rx; 16378 *max_tx = tx; 16379 if (!rx || !tx || !cp) 16380 return -ENOMEM; 16381 16382 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared); 16383} 16384 16385static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx, 16386 bool shared) 16387{ 16388 int rc; 16389 16390 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 16391 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) { 16392 /* Not enough rings, try disabling agg rings. */ 16393 bp->flags &= ~BNXT_FLAG_AGG_RINGS; 16394 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared); 16395 if (rc) { 16396 /* set BNXT_FLAG_AGG_RINGS back for consistency */ 16397 bp->flags |= BNXT_FLAG_AGG_RINGS; 16398 return rc; 16399 } 16400 bp->flags |= BNXT_FLAG_NO_AGG_RINGS; 16401 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 16402 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); 16403 bnxt_set_ring_params(bp); 16404 } 16405 16406 if (bp->flags & BNXT_FLAG_ROCE_CAP) { 16407 int max_cp, max_stat, max_irq; 16408 16409 /* Reserve minimum resources for RoCE */ 16410 max_cp = bnxt_get_max_func_cp_rings(bp); 16411 max_stat = bnxt_get_max_func_stat_ctxs(bp); 16412 max_irq = bnxt_get_max_func_irqs(bp); 16413 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS || 16414 max_irq <= BNXT_MIN_ROCE_CP_RINGS || 16415 max_stat <= BNXT_MIN_ROCE_STAT_CTXS) 16416 return 0; 16417 16418 max_cp -= BNXT_MIN_ROCE_CP_RINGS; 16419 max_irq -= BNXT_MIN_ROCE_CP_RINGS; 16420 max_stat -= BNXT_MIN_ROCE_STAT_CTXS; 16421 max_cp = min_t(int, max_cp, max_irq); 16422 max_cp = min_t(int, max_cp, max_stat); 16423 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared); 16424 if (rc) 16425 rc = 0; 16426 } 16427 return rc; 16428} 16429 16430/* In initial default shared ring setting, each shared ring must have a 16431 * RX/TX ring pair. 16432 */ 16433static void bnxt_trim_dflt_sh_rings(struct bnxt *bp) 16434{ 16435 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings); 16436 bp->rx_nr_rings = bp->cp_nr_rings; 16437 bp->tx_nr_rings_per_tc = bp->cp_nr_rings; 16438 bp->tx_nr_rings = bnxt_tx_nr_rings(bp); 16439} 16440 16441static void bnxt_adj_dflt_rings(struct bnxt *bp, bool sh) 16442{ 16443 if (sh) 16444 bnxt_trim_dflt_sh_rings(bp); 16445 else 16446 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings; 16447 bp->tx_nr_rings = bnxt_tx_nr_rings(bp); 16448 if (sh && READ_ONCE(bp->xdp_prog)) { 16449 bnxt_set_xdp_tx_rings(bp); 16450 bnxt_set_cp_rings(bp, true); 16451 } 16452} 16453 16454static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) 16455{ 16456 int dflt_rings, max_rx_rings, max_tx_rings, rc; 16457 int avail_msix; 16458 16459 if (!bnxt_can_reserve_rings(bp)) 16460 return 0; 16461 16462 if (sh) 16463 bp->flags |= BNXT_FLAG_SHARED_RINGS; 16464 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); 16465 /* Reduce default rings on multi-port cards so that total default 16466 * rings do not exceed CPU count. 16467 */ 16468 if (bp->port_count > 1) { 16469 int max_rings = 16470 max_t(int, num_online_cpus() / bp->port_count, 1); 16471 16472 dflt_rings = min_t(int, dflt_rings, max_rings); 16473 } 16474 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh); 16475 if (rc) 16476 return rc; 16477 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings); 16478 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings); 16479 16480 bnxt_adj_dflt_rings(bp, sh); 16481 16482 avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings; 16483 if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) { 16484 int ulp_num_msix = min(avail_msix, bp->ulp_num_msix_want); 16485 16486 bnxt_set_ulp_msix_num(bp, ulp_num_msix); 16487 bnxt_set_dflt_ulp_stat_ctxs(bp); 16488 } 16489 16490 rc = __bnxt_reserve_rings(bp); 16491 if (rc && rc != -ENODEV) 16492 netdev_warn(bp->dev, "Unable to reserve tx rings\n"); 16493 16494 bnxt_adj_tx_rings(bp); 16495 if (sh) 16496 bnxt_adj_dflt_rings(bp, true); 16497 16498 /* Rings may have been trimmed, re-reserve the trimmed rings. */ 16499 if (bnxt_need_reserve_rings(bp)) { 16500 rc = __bnxt_reserve_rings(bp); 16501 if (rc && rc != -ENODEV) 16502 netdev_warn(bp->dev, "2nd rings reservation failed.\n"); 16503 bnxt_adj_tx_rings(bp); 16504 } 16505 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { 16506 bp->rx_nr_rings++; 16507 bp->cp_nr_rings++; 16508 } 16509 if (rc) { 16510 bp->tx_nr_rings = 0; 16511 bp->rx_nr_rings = 0; 16512 } 16513 return rc; 16514} 16515 16516static int bnxt_init_dflt_ring_mode(struct bnxt *bp) 16517{ 16518 int rc; 16519 16520 if (bp->tx_nr_rings) 16521 return 0; 16522 16523 bnxt_ulp_irq_stop(bp); 16524 bnxt_clear_int_mode(bp); 16525 rc = bnxt_set_dflt_rings(bp, true); 16526 if (rc) { 16527 if (BNXT_VF(bp) && rc == -ENODEV) 16528 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 16529 else 16530 netdev_err(bp->dev, "Not enough rings available.\n"); 16531 goto init_dflt_ring_err; 16532 } 16533 rc = bnxt_init_int_mode(bp); 16534 if (rc) 16535 goto init_dflt_ring_err; 16536 16537 bnxt_adj_tx_rings(bp); 16538 16539 bnxt_set_dflt_rfs(bp); 16540 16541init_dflt_ring_err: 16542 bnxt_ulp_irq_restart(bp, rc); 16543 return rc; 16544} 16545 16546int bnxt_restore_pf_fw_resources(struct bnxt *bp) 16547{ 16548 int rc; 16549 16550 netdev_ops_assert_locked(bp->dev); 16551 bnxt_hwrm_func_qcaps(bp); 16552 16553 if (netif_running(bp->dev)) 16554 __bnxt_close_nic(bp, true, false); 16555 16556 bnxt_ulp_irq_stop(bp); 16557 bnxt_clear_int_mode(bp); 16558 rc = bnxt_init_int_mode(bp); 16559 bnxt_ulp_irq_restart(bp, rc); 16560 16561 if (netif_running(bp->dev)) { 16562 if (rc) 16563 netif_close(bp->dev); 16564 else 16565 rc = bnxt_open_nic(bp, true, false); 16566 } 16567 16568 return rc; 16569} 16570 16571static int bnxt_init_mac_addr(struct bnxt *bp) 16572{ 16573 int rc = 0; 16574 16575 if (BNXT_PF(bp)) { 16576 eth_hw_addr_set(bp->dev, bp->pf.mac_addr); 16577 } else { 16578#ifdef CONFIG_BNXT_SRIOV 16579 struct bnxt_vf_info *vf = &bp->vf; 16580 bool strict_approval = true; 16581 16582 if (is_valid_ether_addr(vf->mac_addr)) { 16583 /* overwrite netdev dev_addr with admin VF MAC */ 16584 eth_hw_addr_set(bp->dev, vf->mac_addr); 16585 /* Older PF driver or firmware may not approve this 16586 * correctly. 16587 */ 16588 strict_approval = false; 16589 } else { 16590 eth_hw_addr_random(bp->dev); 16591 } 16592 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval); 16593#endif 16594 } 16595 return rc; 16596} 16597 16598static void bnxt_vpd_read_info(struct bnxt *bp) 16599{ 16600 struct pci_dev *pdev = bp->pdev; 16601 unsigned int vpd_size, kw_len; 16602 int pos, size; 16603 u8 *vpd_data; 16604 16605 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 16606 if (IS_ERR(vpd_data)) { 16607 pci_warn(pdev, "Unable to read VPD\n"); 16608 return; 16609 } 16610 16611 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 16612 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len); 16613 if (pos < 0) 16614 goto read_sn; 16615 16616 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 16617 memcpy(bp->board_partno, &vpd_data[pos], size); 16618 16619read_sn: 16620 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, 16621 PCI_VPD_RO_KEYWORD_SERIALNO, 16622 &kw_len); 16623 if (pos < 0) 16624 goto exit; 16625 16626 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1); 16627 memcpy(bp->board_serialno, &vpd_data[pos], size); 16628exit: 16629 kfree(vpd_data); 16630} 16631 16632static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[]) 16633{ 16634 struct pci_dev *pdev = bp->pdev; 16635 u64 qword; 16636 16637 qword = pci_get_dsn(pdev); 16638 if (!qword) { 16639 netdev_info(bp->dev, "Unable to read adapter's DSN\n"); 16640 return -EOPNOTSUPP; 16641 } 16642 16643 put_unaligned_le64(qword, dsn); 16644 16645 bp->flags |= BNXT_FLAG_DSN_VALID; 16646 return 0; 16647} 16648 16649static int bnxt_map_db_bar(struct bnxt *bp) 16650{ 16651 if (!bp->db_size) 16652 return -ENODEV; 16653 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size); 16654 if (!bp->bar1) 16655 return -ENOMEM; 16656 return 0; 16657} 16658 16659void bnxt_print_device_info(struct bnxt *bp) 16660{ 16661 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n", 16662 board_info[bp->board_idx].name, 16663 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr); 16664 16665 pcie_print_link_status(bp->pdev); 16666} 16667 16668static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 16669{ 16670 struct bnxt_hw_resc *hw_resc; 16671 struct net_device *dev; 16672 struct bnxt *bp; 16673 int rc, max_irqs; 16674 16675 if (pci_is_bridge(pdev)) 16676 return -ENODEV; 16677 16678 if (!pdev->msix_cap) { 16679 dev_err(&pdev->dev, "MSIX capability not found, aborting\n"); 16680 return -ENODEV; 16681 } 16682 16683 /* Clear any pending DMA transactions from crash kernel 16684 * while loading driver in capture kernel. 16685 */ 16686 if (is_kdump_kernel()) { 16687 pci_clear_master(pdev); 16688 pcie_flr(pdev); 16689 } 16690 16691 max_irqs = bnxt_get_max_irq(pdev); 16692 dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE, 16693 max_irqs); 16694 if (!dev) 16695 return -ENOMEM; 16696 16697 bp = netdev_priv(dev); 16698 bp->board_idx = ent->driver_data; 16699 bp->msg_enable = BNXT_DEF_MSG_ENABLE; 16700 bnxt_set_max_func_irqs(bp, max_irqs); 16701 16702 if (bnxt_vf_pciid(bp->board_idx)) 16703 bp->flags |= BNXT_FLAG_VF; 16704 16705 /* No devlink port registration in case of a VF */ 16706 if (BNXT_PF(bp)) 16707 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port); 16708 16709 rc = bnxt_init_board(pdev, dev); 16710 if (rc < 0) 16711 goto init_err_free; 16712 16713 dev->netdev_ops = &bnxt_netdev_ops; 16714 dev->stat_ops = &bnxt_stat_ops; 16715 dev->watchdog_timeo = BNXT_TX_TIMEOUT; 16716 dev->ethtool_ops = &bnxt_ethtool_ops; 16717 pci_set_drvdata(pdev, dev); 16718 16719 rc = bnxt_alloc_hwrm_resources(bp); 16720 if (rc) 16721 goto init_err_pci_clean; 16722 16723 mutex_init(&bp->hwrm_cmd_lock); 16724 mutex_init(&bp->link_lock); 16725 16726 rc = bnxt_fw_init_one_p1(bp); 16727 if (rc) 16728 goto init_err_pci_clean; 16729 16730 if (BNXT_PF(bp)) 16731 bnxt_vpd_read_info(bp); 16732 16733 if (BNXT_CHIP_P5_PLUS(bp)) { 16734 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS; 16735 if (BNXT_CHIP_P7(bp)) 16736 bp->flags |= BNXT_FLAG_CHIP_P7; 16737 } 16738 16739 rc = bnxt_alloc_rss_indir_tbl(bp); 16740 if (rc) 16741 goto init_err_pci_clean; 16742 16743 rc = bnxt_fw_init_one_p2(bp); 16744 if (rc) 16745 goto init_err_pci_clean; 16746 16747 rc = bnxt_map_db_bar(bp); 16748 if (rc) { 16749 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n", 16750 rc); 16751 goto init_err_pci_clean; 16752 } 16753 16754 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 16755 NETIF_F_TSO | NETIF_F_TSO6 | 16756 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 16757 NETIF_F_GSO_IPXIP4 | 16758 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 16759 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | 16760 NETIF_F_RXCSUM | NETIF_F_GRO; 16761 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) 16762 dev->hw_features |= NETIF_F_GSO_UDP_L4; 16763 16764 if (BNXT_SUPPORTS_TPA(bp)) 16765 dev->hw_features |= NETIF_F_LRO; 16766 16767 dev->hw_enc_features = 16768 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG | 16769 NETIF_F_TSO | NETIF_F_TSO6 | 16770 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 16771 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | 16772 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL; 16773 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) 16774 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 16775 if (bp->flags & BNXT_FLAG_CHIP_P7) 16776 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7; 16777 else 16778 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels; 16779 16780 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM | 16781 NETIF_F_GSO_GRE_CSUM; 16782 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA; 16783 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP) 16784 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX; 16785 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT) 16786 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX; 16787 if (BNXT_SUPPORTS_TPA(bp)) 16788 dev->hw_features |= NETIF_F_GRO_HW; 16789 dev->features |= dev->hw_features | NETIF_F_HIGHDMA; 16790 if (dev->features & NETIF_F_GRO_HW) 16791 dev->features &= ~NETIF_F_LRO; 16792 dev->priv_flags |= IFF_UNICAST_FLT; 16793 16794 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 16795 if (bp->tso_max_segs) 16796 netif_set_tso_max_segs(dev, bp->tso_max_segs); 16797 16798 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 16799 NETDEV_XDP_ACT_RX_SG; 16800 16801#ifdef CONFIG_BNXT_SRIOV 16802 init_waitqueue_head(&bp->sriov_cfg_wait); 16803#endif 16804 if (BNXT_SUPPORTS_TPA(bp)) { 16805 bp->gro_func = bnxt_gro_func_5730x; 16806 if (BNXT_CHIP_P4(bp)) 16807 bp->gro_func = bnxt_gro_func_5731x; 16808 else if (BNXT_CHIP_P5_PLUS(bp)) 16809 bp->gro_func = bnxt_gro_func_5750x; 16810 } 16811 if (!BNXT_CHIP_P4_PLUS(bp)) 16812 bp->flags |= BNXT_FLAG_DOUBLE_DB; 16813 16814 rc = bnxt_init_mac_addr(bp); 16815 if (rc) { 16816 dev_err(&pdev->dev, "Unable to initialize mac address.\n"); 16817 rc = -EADDRNOTAVAIL; 16818 goto init_err_pci_clean; 16819 } 16820 16821 if (BNXT_PF(bp)) { 16822 /* Read the adapter's DSN to use as the eswitch switch_id */ 16823 rc = bnxt_pcie_dsn_get(bp, bp->dsn); 16824 } 16825 16826 /* MTU range: 60 - FW defined max */ 16827 dev->min_mtu = ETH_ZLEN; 16828 dev->max_mtu = bp->max_mtu; 16829 16830 rc = bnxt_probe_phy(bp, true); 16831 if (rc) 16832 goto init_err_pci_clean; 16833 16834 hw_resc = &bp->hw_resc; 16835 bp->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows + 16836 BNXT_L2_FLTR_MAX_FLTR; 16837 /* Older firmware may not report these filters properly */ 16838 if (bp->max_fltr < BNXT_MAX_FLTR) 16839 bp->max_fltr = BNXT_MAX_FLTR; 16840 bnxt_init_l2_fltr_tbl(bp); 16841 __bnxt_set_rx_skb_mode(bp, false); 16842 bnxt_set_tpa_flags(bp); 16843 bnxt_init_ring_params(bp); 16844 bnxt_set_ring_params(bp); 16845 bnxt_rdma_aux_device_init(bp); 16846 rc = bnxt_set_dflt_rings(bp, true); 16847 if (rc) { 16848 if (BNXT_VF(bp) && rc == -ENODEV) { 16849 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n"); 16850 } else { 16851 netdev_err(bp->dev, "Not enough rings available.\n"); 16852 rc = -ENOMEM; 16853 } 16854 goto init_err_pci_clean; 16855 } 16856 16857 bnxt_fw_init_one_p3(bp); 16858 16859 bnxt_init_dflt_coal(bp); 16860 16861 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX) 16862 bp->flags |= BNXT_FLAG_STRIP_VLAN; 16863 16864 rc = bnxt_init_int_mode(bp); 16865 if (rc) 16866 goto init_err_pci_clean; 16867 16868 /* No TC has been set yet and rings may have been trimmed due to 16869 * limited MSIX, so we re-initialize the TX rings per TC. 16870 */ 16871 bp->tx_nr_rings_per_tc = bp->tx_nr_rings; 16872 16873 if (BNXT_PF(bp)) { 16874 if (!bnxt_pf_wq) { 16875 bnxt_pf_wq = 16876 create_singlethread_workqueue("bnxt_pf_wq"); 16877 if (!bnxt_pf_wq) { 16878 dev_err(&pdev->dev, "Unable to create workqueue.\n"); 16879 rc = -ENOMEM; 16880 goto init_err_pci_clean; 16881 } 16882 } 16883 rc = bnxt_init_tc(bp); 16884 if (rc) 16885 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n", 16886 rc); 16887 } 16888 16889 bnxt_inv_fw_health_reg(bp); 16890 rc = bnxt_dl_register(bp); 16891 if (rc) 16892 goto init_err_dl; 16893 16894 INIT_LIST_HEAD(&bp->usr_fltr_list); 16895 16896 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) 16897 bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX; 16898 if (BNXT_SUPPORTS_QUEUE_API(bp)) 16899 dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops; 16900 dev->request_ops_lock = true; 16901 dev->netmem_tx = true; 16902 16903 rc = register_netdev(dev); 16904 if (rc) 16905 goto init_err_cleanup; 16906 16907 bnxt_dl_fw_reporters_create(bp); 16908 16909 bnxt_rdma_aux_device_add(bp); 16910 16911 bnxt_print_device_info(bp); 16912 16913 pci_save_state(pdev); 16914 16915 return 0; 16916init_err_cleanup: 16917 bnxt_rdma_aux_device_uninit(bp); 16918 bnxt_dl_unregister(bp); 16919init_err_dl: 16920 bnxt_shutdown_tc(bp); 16921 bnxt_clear_int_mode(bp); 16922 16923init_err_pci_clean: 16924 bnxt_hwrm_func_drv_unrgtr(bp); 16925 bnxt_ptp_clear(bp); 16926 kfree(bp->ptp_cfg); 16927 bp->ptp_cfg = NULL; 16928 bnxt_free_hwrm_resources(bp); 16929 bnxt_hwmon_uninit(bp); 16930 bnxt_ethtool_free(bp); 16931 kfree(bp->fw_health); 16932 bp->fw_health = NULL; 16933 bnxt_cleanup_pci(bp); 16934 bnxt_free_ctx_mem(bp, true); 16935 bnxt_free_crash_dump_mem(bp); 16936 kfree(bp->rss_indir_tbl); 16937 bp->rss_indir_tbl = NULL; 16938 16939init_err_free: 16940 free_netdev(dev); 16941 return rc; 16942} 16943 16944static void bnxt_shutdown(struct pci_dev *pdev) 16945{ 16946 struct net_device *dev = pci_get_drvdata(pdev); 16947 struct bnxt *bp; 16948 16949 if (!dev) 16950 return; 16951 16952 rtnl_lock(); 16953 netdev_lock(dev); 16954 bp = netdev_priv(dev); 16955 if (!bp) 16956 goto shutdown_exit; 16957 16958 if (netif_running(dev)) 16959 netif_close(dev); 16960 16961 if (bnxt_hwrm_func_drv_unrgtr(bp)) { 16962 pcie_flr(pdev); 16963 goto shutdown_exit; 16964 } 16965 bnxt_ptp_clear(bp); 16966 bnxt_clear_int_mode(bp); 16967 pci_disable_device(pdev); 16968 16969 if (system_state == SYSTEM_POWER_OFF) { 16970 pci_wake_from_d3(pdev, bp->wol); 16971 pci_set_power_state(pdev, PCI_D3hot); 16972 } 16973 16974shutdown_exit: 16975 netdev_unlock(dev); 16976 rtnl_unlock(); 16977} 16978 16979#ifdef CONFIG_PM_SLEEP 16980static int bnxt_suspend(struct device *device) 16981{ 16982 struct net_device *dev = dev_get_drvdata(device); 16983 struct bnxt *bp = netdev_priv(dev); 16984 int rc = 0; 16985 16986 bnxt_ulp_stop(bp); 16987 16988 netdev_lock(dev); 16989 if (netif_running(dev)) { 16990 netif_device_detach(dev); 16991 rc = bnxt_close(dev); 16992 } 16993 bnxt_hwrm_func_drv_unrgtr(bp); 16994 bnxt_ptp_clear(bp); 16995 pci_disable_device(bp->pdev); 16996 bnxt_free_ctx_mem(bp, false); 16997 netdev_unlock(dev); 16998 return rc; 16999} 17000 17001static int bnxt_resume(struct device *device) 17002{ 17003 struct net_device *dev = dev_get_drvdata(device); 17004 struct bnxt *bp = netdev_priv(dev); 17005 int rc = 0; 17006 17007 netdev_lock(dev); 17008 rc = pci_enable_device(bp->pdev); 17009 if (rc) { 17010 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n", 17011 rc); 17012 goto resume_exit; 17013 } 17014 pci_set_master(bp->pdev); 17015 if (bnxt_hwrm_ver_get(bp)) { 17016 rc = -ENODEV; 17017 goto resume_exit; 17018 } 17019 rc = bnxt_hwrm_func_reset(bp); 17020 if (rc) { 17021 rc = -EBUSY; 17022 goto resume_exit; 17023 } 17024 17025 rc = bnxt_hwrm_func_qcaps(bp); 17026 if (rc) 17027 goto resume_exit; 17028 17029 bnxt_clear_reservations(bp, true); 17030 17031 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) { 17032 rc = -ENODEV; 17033 goto resume_exit; 17034 } 17035 if (bp->fw_crash_mem) 17036 bnxt_hwrm_crash_dump_mem_cfg(bp); 17037 17038 if (bnxt_ptp_init(bp)) { 17039 kfree(bp->ptp_cfg); 17040 bp->ptp_cfg = NULL; 17041 } 17042 bnxt_get_wol_settings(bp); 17043 if (netif_running(dev)) { 17044 rc = bnxt_open(dev); 17045 if (!rc) 17046 netif_device_attach(dev); 17047 } 17048 17049resume_exit: 17050 netdev_unlock(bp->dev); 17051 bnxt_ulp_start(bp, rc); 17052 if (!rc) 17053 bnxt_reenable_sriov(bp); 17054 return rc; 17055} 17056 17057static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume); 17058#define BNXT_PM_OPS (&bnxt_pm_ops) 17059 17060#else 17061 17062#define BNXT_PM_OPS NULL 17063 17064#endif /* CONFIG_PM_SLEEP */ 17065 17066/** 17067 * bnxt_io_error_detected - called when PCI error is detected 17068 * @pdev: Pointer to PCI device 17069 * @state: The current pci connection state 17070 * 17071 * This function is called after a PCI bus error affecting 17072 * this device has been detected. 17073 */ 17074static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, 17075 pci_channel_state_t state) 17076{ 17077 struct net_device *netdev = pci_get_drvdata(pdev); 17078 struct bnxt *bp = netdev_priv(netdev); 17079 bool abort = false; 17080 17081 netdev_info(netdev, "PCI I/O error detected\n"); 17082 17083 bnxt_ulp_stop(bp); 17084 17085 netdev_lock(netdev); 17086 netif_device_detach(netdev); 17087 17088 if (test_and_set_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { 17089 netdev_err(bp->dev, "Firmware reset already in progress\n"); 17090 abort = true; 17091 } 17092 17093 if (abort || state == pci_channel_io_perm_failure) { 17094 netdev_unlock(netdev); 17095 return PCI_ERS_RESULT_DISCONNECT; 17096 } 17097 17098 /* Link is not reliable anymore if state is pci_channel_io_frozen 17099 * so we disable bus master to prevent any potential bad DMAs before 17100 * freeing kernel memory. 17101 */ 17102 if (state == pci_channel_io_frozen) { 17103 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); 17104 bnxt_fw_fatal_close(bp); 17105 } 17106 17107 if (netif_running(netdev)) 17108 __bnxt_close_nic(bp, true, true); 17109 17110 if (pci_is_enabled(pdev)) 17111 pci_disable_device(pdev); 17112 bnxt_free_ctx_mem(bp, false); 17113 netdev_unlock(netdev); 17114 17115 /* Request a slot reset. */ 17116 return PCI_ERS_RESULT_NEED_RESET; 17117} 17118 17119/** 17120 * bnxt_io_slot_reset - called after the pci bus has been reset. 17121 * @pdev: Pointer to PCI device 17122 * 17123 * Restart the card from scratch, as if from a cold-boot. 17124 * At this point, the card has experienced a hard reset, 17125 * followed by fixups by BIOS, and has its config space 17126 * set up identically to what it was at cold boot. 17127 */ 17128static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) 17129{ 17130 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; 17131 struct net_device *netdev = pci_get_drvdata(pdev); 17132 struct bnxt *bp = netdev_priv(netdev); 17133 int retry = 0; 17134 int err = 0; 17135 int off; 17136 17137 netdev_info(bp->dev, "PCI Slot Reset\n"); 17138 17139 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && 17140 test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) 17141 msleep(900); 17142 17143 netdev_lock(netdev); 17144 17145 if (pci_enable_device(pdev)) { 17146 dev_err(&pdev->dev, 17147 "Cannot re-enable PCI device after reset.\n"); 17148 } else { 17149 pci_set_master(pdev); 17150 /* Upon fatal error, our device internal logic that latches to 17151 * BAR value is getting reset and will restore only upon 17152 * rewriting the BARs. 17153 * 17154 * As pci_restore_state() does not re-write the BARs if the 17155 * value is same as saved value earlier, driver needs to 17156 * write the BARs to 0 to force restore, in case of fatal error. 17157 */ 17158 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, 17159 &bp->state)) { 17160 for (off = PCI_BASE_ADDRESS_0; 17161 off <= PCI_BASE_ADDRESS_5; off += 4) 17162 pci_write_config_dword(bp->pdev, off, 0); 17163 } 17164 pci_restore_state(pdev); 17165 pci_save_state(pdev); 17166 17167 bnxt_inv_fw_health_reg(bp); 17168 bnxt_try_map_fw_health_reg(bp); 17169 17170 /* In some PCIe AER scenarios, firmware may take up to 17171 * 10 seconds to become ready in the worst case. 17172 */ 17173 do { 17174 err = bnxt_try_recover_fw(bp); 17175 if (!err) 17176 break; 17177 retry++; 17178 } while (retry < BNXT_FW_SLOT_RESET_RETRY); 17179 17180 if (err) { 17181 dev_err(&pdev->dev, "Firmware not ready\n"); 17182 goto reset_exit; 17183 } 17184 17185 err = bnxt_hwrm_func_reset(bp); 17186 if (!err) 17187 result = PCI_ERS_RESULT_RECOVERED; 17188 17189 /* IRQ will be initialized later in bnxt_io_resume */ 17190 bnxt_ulp_irq_stop(bp); 17191 bnxt_clear_int_mode(bp); 17192 } 17193 17194reset_exit: 17195 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); 17196 bnxt_clear_reservations(bp, true); 17197 netdev_unlock(netdev); 17198 17199 return result; 17200} 17201 17202/** 17203 * bnxt_io_resume - called when traffic can start flowing again. 17204 * @pdev: Pointer to PCI device 17205 * 17206 * This callback is called when the error recovery driver tells 17207 * us that its OK to resume normal operation. 17208 */ 17209static void bnxt_io_resume(struct pci_dev *pdev) 17210{ 17211 struct net_device *netdev = pci_get_drvdata(pdev); 17212 struct bnxt *bp = netdev_priv(netdev); 17213 int err; 17214 17215 netdev_info(bp->dev, "PCI Slot Resume\n"); 17216 netdev_lock(netdev); 17217 17218 err = bnxt_hwrm_func_qcaps(bp); 17219 if (!err) { 17220 if (netif_running(netdev)) { 17221 err = bnxt_open(netdev); 17222 } else { 17223 err = bnxt_reserve_rings(bp, true); 17224 if (!err) 17225 err = bnxt_init_int_mode(bp); 17226 } 17227 } 17228 17229 if (!err) 17230 netif_device_attach(netdev); 17231 17232 netdev_unlock(netdev); 17233 bnxt_ulp_start(bp, err); 17234 if (!err) 17235 bnxt_reenable_sriov(bp); 17236} 17237 17238static const struct pci_error_handlers bnxt_err_handler = { 17239 .error_detected = bnxt_io_error_detected, 17240 .slot_reset = bnxt_io_slot_reset, 17241 .resume = bnxt_io_resume 17242}; 17243 17244static struct pci_driver bnxt_pci_driver = { 17245 .name = DRV_MODULE_NAME, 17246 .id_table = bnxt_pci_tbl, 17247 .probe = bnxt_init_one, 17248 .remove = bnxt_remove_one, 17249 .shutdown = bnxt_shutdown, 17250 .driver.pm = BNXT_PM_OPS, 17251 .err_handler = &bnxt_err_handler, 17252#if defined(CONFIG_BNXT_SRIOV) 17253 .sriov_configure = bnxt_sriov_configure, 17254#endif 17255}; 17256 17257static int __init bnxt_init(void) 17258{ 17259 int err; 17260 17261 bnxt_debug_init(); 17262 err = pci_register_driver(&bnxt_pci_driver); 17263 if (err) { 17264 bnxt_debug_exit(); 17265 return err; 17266 } 17267 17268 return 0; 17269} 17270 17271static void __exit bnxt_exit(void) 17272{ 17273 pci_unregister_driver(&bnxt_pci_driver); 17274 if (bnxt_pf_wq) 17275 destroy_workqueue(bnxt_pf_wq); 17276 bnxt_debug_exit(); 17277} 17278 17279module_init(bnxt_init); 17280module_exit(bnxt_exit);
~/
Read-only file preview.