FILE
hns3_enet.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
1// SPDX-License-Identifier: GPL-2.0+ 2// Copyright (c) 2016-2017 Hisilicon Limited. 3 4#include <linux/dma-mapping.h> 5#include <linux/etherdevice.h> 6#include <linux/interrupt.h> 7#ifdef CONFIG_RFS_ACCEL 8#include <linux/cpu_rmap.h> 9#endif 10#include <linux/if_vlan.h> 11#include <linux/irq.h> 12#include <linux/ip.h> 13#include <linux/ipv6.h> 14#include <linux/iommu.h> 15#include <linux/module.h> 16#include <linux/pci.h> 17#include <linux/skbuff.h> 18#include <linux/sctp.h> 19#include <net/gre.h> 20#include <net/gro.h> 21#include <net/ip6_checksum.h> 22#include <net/page_pool/helpers.h> 23#include <net/pkt_cls.h> 24#include <net/pkt_sched.h> 25#include <net/tcp.h> 26#include <net/vxlan.h> 27#include <net/geneve.h> 28 29#include "hnae3.h" 30#include "hns3_enet.h" 31/* All hns3 tracepoints are defined by the include below, which 32 * must be included exactly once across the whole kernel with 33 * CREATE_TRACE_POINTS defined 34 */ 35#define CREATE_TRACE_POINTS 36#include "hns3_trace.h" 37 38#define hns3_set_field(origin, shift, val) ((origin) |= (val) << (shift)) 39#define hns3_tx_bd_count(S) DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE) 40 41#define hns3_rl_err(fmt, ...) \ 42 do { \ 43 if (net_ratelimit()) \ 44 netdev_err(fmt, ##__VA_ARGS__); \ 45 } while (0) 46 47static void hns3_clear_all_ring(struct hnae3_handle *h, bool force); 48 49static const char hns3_driver_name[] = "hns3"; 50static const char hns3_driver_string[] = 51 "Hisilicon Ethernet Network Driver for Hip08 Family"; 52static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation."; 53static struct hnae3_client client; 54 55static int debug = -1; 56module_param(debug, int, 0); 57MODULE_PARM_DESC(debug, " Network interface message level setting"); 58 59static unsigned int tx_sgl = 1; 60module_param(tx_sgl, uint, 0600); 61MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping"); 62 63static bool page_pool_enabled = true; 64module_param(page_pool_enabled, bool, 0400); 65 66#define HNS3_SGL_SIZE(nfrag) (sizeof(struct scatterlist) * (nfrag) + \ 67 sizeof(struct sg_table)) 68#define HNS3_MAX_SGL_SIZE ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM), \ 69 dma_get_cache_alignment()) 70 71#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 72 NETIF_MSG_IFDOWN | NETIF_MSG_IFUP) 73 74#define HNS3_INNER_VLAN_TAG 1 75#define HNS3_OUTER_VLAN_TAG 2 76 77#define HNS3_MIN_TX_LEN 33U 78#define HNS3_MIN_TUN_PKT_LEN 65U 79 80/* hns3_pci_tbl - PCI Device ID Table 81 * 82 * Last entry must be all 0s 83 * 84 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 85 * Class, Class Mask, private data (not used) } 86 */ 87static const struct pci_device_id hns3_pci_tbl[] = { 88 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0}, 89 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0}, 90 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 91 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 92 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 93 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 94 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 95 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 96 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 97 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 98 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 99 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 100 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 101 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 102 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0}, 103 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF), 104 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS}, 105 /* required last entry */ 106 {0,} 107}; 108MODULE_DEVICE_TABLE(pci, hns3_pci_tbl); 109 110#define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t, h) \ 111 { ptype, \ 112 l, \ 113 CHECKSUM_##s, \ 114 HNS3_L3_TYPE_##t, \ 115 1, \ 116 h} 117 118#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \ 119 { ptype, 0, CHECKSUM_NONE, HNS3_L3_TYPE_PARSE_FAIL, 0, \ 120 PKT_HASH_TYPE_NONE } 121 122static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = { 123 HNS3_RX_PTYPE_UNUSED_ENTRY(0), 124 HNS3_RX_PTYPE_ENTRY(1, 0, COMPLETE, ARP, PKT_HASH_TYPE_NONE), 125 HNS3_RX_PTYPE_ENTRY(2, 0, COMPLETE, RARP, PKT_HASH_TYPE_NONE), 126 HNS3_RX_PTYPE_ENTRY(3, 0, COMPLETE, LLDP, PKT_HASH_TYPE_NONE), 127 HNS3_RX_PTYPE_ENTRY(4, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 128 HNS3_RX_PTYPE_ENTRY(5, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 129 HNS3_RX_PTYPE_ENTRY(6, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 130 HNS3_RX_PTYPE_ENTRY(7, 0, COMPLETE, CNM, PKT_HASH_TYPE_NONE), 131 HNS3_RX_PTYPE_ENTRY(8, 0, NONE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 132 HNS3_RX_PTYPE_UNUSED_ENTRY(9), 133 HNS3_RX_PTYPE_UNUSED_ENTRY(10), 134 HNS3_RX_PTYPE_UNUSED_ENTRY(11), 135 HNS3_RX_PTYPE_UNUSED_ENTRY(12), 136 HNS3_RX_PTYPE_UNUSED_ENTRY(13), 137 HNS3_RX_PTYPE_UNUSED_ENTRY(14), 138 HNS3_RX_PTYPE_UNUSED_ENTRY(15), 139 HNS3_RX_PTYPE_ENTRY(16, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 140 HNS3_RX_PTYPE_ENTRY(17, 0, COMPLETE, IPV4, PKT_HASH_TYPE_NONE), 141 HNS3_RX_PTYPE_ENTRY(18, 0, COMPLETE, IPV4, PKT_HASH_TYPE_NONE), 142 HNS3_RX_PTYPE_ENTRY(19, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 143 HNS3_RX_PTYPE_ENTRY(20, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 144 HNS3_RX_PTYPE_ENTRY(21, 0, NONE, IPV4, PKT_HASH_TYPE_NONE), 145 HNS3_RX_PTYPE_ENTRY(22, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 146 HNS3_RX_PTYPE_ENTRY(23, 0, NONE, IPV4, PKT_HASH_TYPE_L3), 147 HNS3_RX_PTYPE_ENTRY(24, 0, NONE, IPV4, PKT_HASH_TYPE_L3), 148 HNS3_RX_PTYPE_ENTRY(25, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 149 HNS3_RX_PTYPE_UNUSED_ENTRY(26), 150 HNS3_RX_PTYPE_UNUSED_ENTRY(27), 151 HNS3_RX_PTYPE_UNUSED_ENTRY(28), 152 HNS3_RX_PTYPE_ENTRY(29, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 153 HNS3_RX_PTYPE_ENTRY(30, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 154 HNS3_RX_PTYPE_ENTRY(31, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 155 HNS3_RX_PTYPE_ENTRY(32, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 156 HNS3_RX_PTYPE_ENTRY(33, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 157 HNS3_RX_PTYPE_ENTRY(34, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 158 HNS3_RX_PTYPE_ENTRY(35, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 159 HNS3_RX_PTYPE_ENTRY(36, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 160 HNS3_RX_PTYPE_ENTRY(37, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 161 HNS3_RX_PTYPE_UNUSED_ENTRY(38), 162 HNS3_RX_PTYPE_ENTRY(39, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 163 HNS3_RX_PTYPE_ENTRY(40, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 164 HNS3_RX_PTYPE_ENTRY(41, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 165 HNS3_RX_PTYPE_ENTRY(42, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 166 HNS3_RX_PTYPE_ENTRY(43, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 167 HNS3_RX_PTYPE_ENTRY(44, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 168 HNS3_RX_PTYPE_ENTRY(45, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 169 HNS3_RX_PTYPE_UNUSED_ENTRY(46), 170 HNS3_RX_PTYPE_UNUSED_ENTRY(47), 171 HNS3_RX_PTYPE_UNUSED_ENTRY(48), 172 HNS3_RX_PTYPE_UNUSED_ENTRY(49), 173 HNS3_RX_PTYPE_UNUSED_ENTRY(50), 174 HNS3_RX_PTYPE_UNUSED_ENTRY(51), 175 HNS3_RX_PTYPE_UNUSED_ENTRY(52), 176 HNS3_RX_PTYPE_UNUSED_ENTRY(53), 177 HNS3_RX_PTYPE_UNUSED_ENTRY(54), 178 HNS3_RX_PTYPE_UNUSED_ENTRY(55), 179 HNS3_RX_PTYPE_UNUSED_ENTRY(56), 180 HNS3_RX_PTYPE_UNUSED_ENTRY(57), 181 HNS3_RX_PTYPE_UNUSED_ENTRY(58), 182 HNS3_RX_PTYPE_UNUSED_ENTRY(59), 183 HNS3_RX_PTYPE_UNUSED_ENTRY(60), 184 HNS3_RX_PTYPE_UNUSED_ENTRY(61), 185 HNS3_RX_PTYPE_UNUSED_ENTRY(62), 186 HNS3_RX_PTYPE_UNUSED_ENTRY(63), 187 HNS3_RX_PTYPE_UNUSED_ENTRY(64), 188 HNS3_RX_PTYPE_UNUSED_ENTRY(65), 189 HNS3_RX_PTYPE_UNUSED_ENTRY(66), 190 HNS3_RX_PTYPE_UNUSED_ENTRY(67), 191 HNS3_RX_PTYPE_UNUSED_ENTRY(68), 192 HNS3_RX_PTYPE_UNUSED_ENTRY(69), 193 HNS3_RX_PTYPE_UNUSED_ENTRY(70), 194 HNS3_RX_PTYPE_UNUSED_ENTRY(71), 195 HNS3_RX_PTYPE_UNUSED_ENTRY(72), 196 HNS3_RX_PTYPE_UNUSED_ENTRY(73), 197 HNS3_RX_PTYPE_UNUSED_ENTRY(74), 198 HNS3_RX_PTYPE_UNUSED_ENTRY(75), 199 HNS3_RX_PTYPE_UNUSED_ENTRY(76), 200 HNS3_RX_PTYPE_UNUSED_ENTRY(77), 201 HNS3_RX_PTYPE_UNUSED_ENTRY(78), 202 HNS3_RX_PTYPE_UNUSED_ENTRY(79), 203 HNS3_RX_PTYPE_UNUSED_ENTRY(80), 204 HNS3_RX_PTYPE_UNUSED_ENTRY(81), 205 HNS3_RX_PTYPE_UNUSED_ENTRY(82), 206 HNS3_RX_PTYPE_UNUSED_ENTRY(83), 207 HNS3_RX_PTYPE_UNUSED_ENTRY(84), 208 HNS3_RX_PTYPE_UNUSED_ENTRY(85), 209 HNS3_RX_PTYPE_UNUSED_ENTRY(86), 210 HNS3_RX_PTYPE_UNUSED_ENTRY(87), 211 HNS3_RX_PTYPE_UNUSED_ENTRY(88), 212 HNS3_RX_PTYPE_UNUSED_ENTRY(89), 213 HNS3_RX_PTYPE_UNUSED_ENTRY(90), 214 HNS3_RX_PTYPE_UNUSED_ENTRY(91), 215 HNS3_RX_PTYPE_UNUSED_ENTRY(92), 216 HNS3_RX_PTYPE_UNUSED_ENTRY(93), 217 HNS3_RX_PTYPE_UNUSED_ENTRY(94), 218 HNS3_RX_PTYPE_UNUSED_ENTRY(95), 219 HNS3_RX_PTYPE_UNUSED_ENTRY(96), 220 HNS3_RX_PTYPE_UNUSED_ENTRY(97), 221 HNS3_RX_PTYPE_UNUSED_ENTRY(98), 222 HNS3_RX_PTYPE_UNUSED_ENTRY(99), 223 HNS3_RX_PTYPE_UNUSED_ENTRY(100), 224 HNS3_RX_PTYPE_UNUSED_ENTRY(101), 225 HNS3_RX_PTYPE_UNUSED_ENTRY(102), 226 HNS3_RX_PTYPE_UNUSED_ENTRY(103), 227 HNS3_RX_PTYPE_UNUSED_ENTRY(104), 228 HNS3_RX_PTYPE_UNUSED_ENTRY(105), 229 HNS3_RX_PTYPE_UNUSED_ENTRY(106), 230 HNS3_RX_PTYPE_UNUSED_ENTRY(107), 231 HNS3_RX_PTYPE_UNUSED_ENTRY(108), 232 HNS3_RX_PTYPE_UNUSED_ENTRY(109), 233 HNS3_RX_PTYPE_UNUSED_ENTRY(110), 234 HNS3_RX_PTYPE_ENTRY(111, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 235 HNS3_RX_PTYPE_ENTRY(112, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 236 HNS3_RX_PTYPE_ENTRY(113, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 237 HNS3_RX_PTYPE_ENTRY(114, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 238 HNS3_RX_PTYPE_ENTRY(115, 0, NONE, IPV6, PKT_HASH_TYPE_L3), 239 HNS3_RX_PTYPE_ENTRY(116, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 240 HNS3_RX_PTYPE_ENTRY(117, 0, NONE, IPV6, PKT_HASH_TYPE_L3), 241 HNS3_RX_PTYPE_ENTRY(118, 0, NONE, IPV6, PKT_HASH_TYPE_L3), 242 HNS3_RX_PTYPE_ENTRY(119, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 243 HNS3_RX_PTYPE_UNUSED_ENTRY(120), 244 HNS3_RX_PTYPE_UNUSED_ENTRY(121), 245 HNS3_RX_PTYPE_UNUSED_ENTRY(122), 246 HNS3_RX_PTYPE_ENTRY(123, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 247 HNS3_RX_PTYPE_ENTRY(124, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE), 248 HNS3_RX_PTYPE_ENTRY(125, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 249 HNS3_RX_PTYPE_ENTRY(126, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 250 HNS3_RX_PTYPE_ENTRY(127, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 251 HNS3_RX_PTYPE_ENTRY(128, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 252 HNS3_RX_PTYPE_ENTRY(129, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4), 253 HNS3_RX_PTYPE_ENTRY(130, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 254 HNS3_RX_PTYPE_ENTRY(131, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3), 255 HNS3_RX_PTYPE_UNUSED_ENTRY(132), 256 HNS3_RX_PTYPE_ENTRY(133, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 257 HNS3_RX_PTYPE_ENTRY(134, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 258 HNS3_RX_PTYPE_ENTRY(135, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 259 HNS3_RX_PTYPE_ENTRY(136, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 260 HNS3_RX_PTYPE_ENTRY(137, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4), 261 HNS3_RX_PTYPE_ENTRY(138, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 262 HNS3_RX_PTYPE_ENTRY(139, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3), 263 HNS3_RX_PTYPE_UNUSED_ENTRY(140), 264 HNS3_RX_PTYPE_UNUSED_ENTRY(141), 265 HNS3_RX_PTYPE_UNUSED_ENTRY(142), 266 HNS3_RX_PTYPE_UNUSED_ENTRY(143), 267 HNS3_RX_PTYPE_UNUSED_ENTRY(144), 268 HNS3_RX_PTYPE_UNUSED_ENTRY(145), 269 HNS3_RX_PTYPE_UNUSED_ENTRY(146), 270 HNS3_RX_PTYPE_UNUSED_ENTRY(147), 271 HNS3_RX_PTYPE_UNUSED_ENTRY(148), 272 HNS3_RX_PTYPE_UNUSED_ENTRY(149), 273 HNS3_RX_PTYPE_UNUSED_ENTRY(150), 274 HNS3_RX_PTYPE_UNUSED_ENTRY(151), 275 HNS3_RX_PTYPE_UNUSED_ENTRY(152), 276 HNS3_RX_PTYPE_UNUSED_ENTRY(153), 277 HNS3_RX_PTYPE_UNUSED_ENTRY(154), 278 HNS3_RX_PTYPE_UNUSED_ENTRY(155), 279 HNS3_RX_PTYPE_UNUSED_ENTRY(156), 280 HNS3_RX_PTYPE_UNUSED_ENTRY(157), 281 HNS3_RX_PTYPE_UNUSED_ENTRY(158), 282 HNS3_RX_PTYPE_UNUSED_ENTRY(159), 283 HNS3_RX_PTYPE_UNUSED_ENTRY(160), 284 HNS3_RX_PTYPE_UNUSED_ENTRY(161), 285 HNS3_RX_PTYPE_UNUSED_ENTRY(162), 286 HNS3_RX_PTYPE_UNUSED_ENTRY(163), 287 HNS3_RX_PTYPE_UNUSED_ENTRY(164), 288 HNS3_RX_PTYPE_UNUSED_ENTRY(165), 289 HNS3_RX_PTYPE_UNUSED_ENTRY(166), 290 HNS3_RX_PTYPE_UNUSED_ENTRY(167), 291 HNS3_RX_PTYPE_UNUSED_ENTRY(168), 292 HNS3_RX_PTYPE_UNUSED_ENTRY(169), 293 HNS3_RX_PTYPE_UNUSED_ENTRY(170), 294 HNS3_RX_PTYPE_UNUSED_ENTRY(171), 295 HNS3_RX_PTYPE_UNUSED_ENTRY(172), 296 HNS3_RX_PTYPE_UNUSED_ENTRY(173), 297 HNS3_RX_PTYPE_UNUSED_ENTRY(174), 298 HNS3_RX_PTYPE_UNUSED_ENTRY(175), 299 HNS3_RX_PTYPE_UNUSED_ENTRY(176), 300 HNS3_RX_PTYPE_UNUSED_ENTRY(177), 301 HNS3_RX_PTYPE_UNUSED_ENTRY(178), 302 HNS3_RX_PTYPE_UNUSED_ENTRY(179), 303 HNS3_RX_PTYPE_UNUSED_ENTRY(180), 304 HNS3_RX_PTYPE_UNUSED_ENTRY(181), 305 HNS3_RX_PTYPE_UNUSED_ENTRY(182), 306 HNS3_RX_PTYPE_UNUSED_ENTRY(183), 307 HNS3_RX_PTYPE_UNUSED_ENTRY(184), 308 HNS3_RX_PTYPE_UNUSED_ENTRY(185), 309 HNS3_RX_PTYPE_UNUSED_ENTRY(186), 310 HNS3_RX_PTYPE_UNUSED_ENTRY(187), 311 HNS3_RX_PTYPE_UNUSED_ENTRY(188), 312 HNS3_RX_PTYPE_UNUSED_ENTRY(189), 313 HNS3_RX_PTYPE_UNUSED_ENTRY(190), 314 HNS3_RX_PTYPE_UNUSED_ENTRY(191), 315 HNS3_RX_PTYPE_UNUSED_ENTRY(192), 316 HNS3_RX_PTYPE_UNUSED_ENTRY(193), 317 HNS3_RX_PTYPE_UNUSED_ENTRY(194), 318 HNS3_RX_PTYPE_UNUSED_ENTRY(195), 319 HNS3_RX_PTYPE_UNUSED_ENTRY(196), 320 HNS3_RX_PTYPE_UNUSED_ENTRY(197), 321 HNS3_RX_PTYPE_UNUSED_ENTRY(198), 322 HNS3_RX_PTYPE_UNUSED_ENTRY(199), 323 HNS3_RX_PTYPE_UNUSED_ENTRY(200), 324 HNS3_RX_PTYPE_UNUSED_ENTRY(201), 325 HNS3_RX_PTYPE_UNUSED_ENTRY(202), 326 HNS3_RX_PTYPE_UNUSED_ENTRY(203), 327 HNS3_RX_PTYPE_UNUSED_ENTRY(204), 328 HNS3_RX_PTYPE_UNUSED_ENTRY(205), 329 HNS3_RX_PTYPE_UNUSED_ENTRY(206), 330 HNS3_RX_PTYPE_UNUSED_ENTRY(207), 331 HNS3_RX_PTYPE_UNUSED_ENTRY(208), 332 HNS3_RX_PTYPE_UNUSED_ENTRY(209), 333 HNS3_RX_PTYPE_UNUSED_ENTRY(210), 334 HNS3_RX_PTYPE_UNUSED_ENTRY(211), 335 HNS3_RX_PTYPE_UNUSED_ENTRY(212), 336 HNS3_RX_PTYPE_UNUSED_ENTRY(213), 337 HNS3_RX_PTYPE_UNUSED_ENTRY(214), 338 HNS3_RX_PTYPE_UNUSED_ENTRY(215), 339 HNS3_RX_PTYPE_UNUSED_ENTRY(216), 340 HNS3_RX_PTYPE_UNUSED_ENTRY(217), 341 HNS3_RX_PTYPE_UNUSED_ENTRY(218), 342 HNS3_RX_PTYPE_UNUSED_ENTRY(219), 343 HNS3_RX_PTYPE_UNUSED_ENTRY(220), 344 HNS3_RX_PTYPE_UNUSED_ENTRY(221), 345 HNS3_RX_PTYPE_UNUSED_ENTRY(222), 346 HNS3_RX_PTYPE_UNUSED_ENTRY(223), 347 HNS3_RX_PTYPE_UNUSED_ENTRY(224), 348 HNS3_RX_PTYPE_UNUSED_ENTRY(225), 349 HNS3_RX_PTYPE_UNUSED_ENTRY(226), 350 HNS3_RX_PTYPE_UNUSED_ENTRY(227), 351 HNS3_RX_PTYPE_UNUSED_ENTRY(228), 352 HNS3_RX_PTYPE_UNUSED_ENTRY(229), 353 HNS3_RX_PTYPE_UNUSED_ENTRY(230), 354 HNS3_RX_PTYPE_UNUSED_ENTRY(231), 355 HNS3_RX_PTYPE_UNUSED_ENTRY(232), 356 HNS3_RX_PTYPE_UNUSED_ENTRY(233), 357 HNS3_RX_PTYPE_UNUSED_ENTRY(234), 358 HNS3_RX_PTYPE_UNUSED_ENTRY(235), 359 HNS3_RX_PTYPE_UNUSED_ENTRY(236), 360 HNS3_RX_PTYPE_UNUSED_ENTRY(237), 361 HNS3_RX_PTYPE_UNUSED_ENTRY(238), 362 HNS3_RX_PTYPE_UNUSED_ENTRY(239), 363 HNS3_RX_PTYPE_UNUSED_ENTRY(240), 364 HNS3_RX_PTYPE_UNUSED_ENTRY(241), 365 HNS3_RX_PTYPE_UNUSED_ENTRY(242), 366 HNS3_RX_PTYPE_UNUSED_ENTRY(243), 367 HNS3_RX_PTYPE_UNUSED_ENTRY(244), 368 HNS3_RX_PTYPE_UNUSED_ENTRY(245), 369 HNS3_RX_PTYPE_UNUSED_ENTRY(246), 370 HNS3_RX_PTYPE_UNUSED_ENTRY(247), 371 HNS3_RX_PTYPE_UNUSED_ENTRY(248), 372 HNS3_RX_PTYPE_UNUSED_ENTRY(249), 373 HNS3_RX_PTYPE_UNUSED_ENTRY(250), 374 HNS3_RX_PTYPE_UNUSED_ENTRY(251), 375 HNS3_RX_PTYPE_UNUSED_ENTRY(252), 376 HNS3_RX_PTYPE_UNUSED_ENTRY(253), 377 HNS3_RX_PTYPE_UNUSED_ENTRY(254), 378 HNS3_RX_PTYPE_UNUSED_ENTRY(255), 379}; 380 381#define HNS3_INVALID_PTYPE \ 382 ARRAY_SIZE(hns3_rx_ptype_tbl) 383 384static irqreturn_t hns3_irq_handle(int irq, void *vector) 385{ 386 struct hns3_enet_tqp_vector *tqp_vector = vector; 387 388 napi_schedule_irqoff(&tqp_vector->napi); 389 tqp_vector->event_cnt++; 390 391 return IRQ_HANDLED; 392} 393 394static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv) 395{ 396 struct hns3_enet_tqp_vector *tqp_vectors; 397 unsigned int i; 398 399 for (i = 0; i < priv->vector_num; i++) { 400 tqp_vectors = &priv->tqp_vector[i]; 401 402 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED) 403 continue; 404 405 /* clear the affinity mask */ 406 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL); 407 408 /* release the irq resource */ 409 free_irq(tqp_vectors->vector_irq, tqp_vectors); 410 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED; 411 } 412} 413 414static int hns3_nic_init_irq(struct hns3_nic_priv *priv) 415{ 416 struct hns3_enet_tqp_vector *tqp_vectors; 417 int txrx_int_idx = 0; 418 int rx_int_idx = 0; 419 int tx_int_idx = 0; 420 unsigned int i; 421 int ret; 422 423 for (i = 0; i < priv->vector_num; i++) { 424 tqp_vectors = &priv->tqp_vector[i]; 425 426 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED) 427 continue; 428 429 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) { 430 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, 431 "%s-%s-%s-%d", hns3_driver_name, 432 pci_name(priv->ae_handle->pdev), 433 "TxRx", txrx_int_idx++); 434 txrx_int_idx++; 435 } else if (tqp_vectors->rx_group.ring) { 436 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, 437 "%s-%s-%s-%d", hns3_driver_name, 438 pci_name(priv->ae_handle->pdev), 439 "Rx", rx_int_idx++); 440 } else if (tqp_vectors->tx_group.ring) { 441 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN, 442 "%s-%s-%s-%d", hns3_driver_name, 443 pci_name(priv->ae_handle->pdev), 444 "Tx", tx_int_idx++); 445 } else { 446 /* Skip this unused q_vector */ 447 continue; 448 } 449 450 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0'; 451 452 irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN); 453 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0, 454 tqp_vectors->name, tqp_vectors); 455 if (ret) { 456 netdev_err(priv->netdev, "request irq(%d) fail\n", 457 tqp_vectors->vector_irq); 458 hns3_nic_uninit_irq(priv); 459 return ret; 460 } 461 462 irq_set_affinity_hint(tqp_vectors->vector_irq, 463 &tqp_vectors->affinity_mask); 464 465 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED; 466 } 467 468 return 0; 469} 470 471static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector, 472 u32 mask_en) 473{ 474 writel(mask_en, tqp_vector->mask_addr); 475} 476 477static void hns3_irq_enable(struct hns3_enet_tqp_vector *tqp_vector) 478{ 479 napi_enable(&tqp_vector->napi); 480 enable_irq(tqp_vector->vector_irq); 481} 482 483static void hns3_irq_disable(struct hns3_enet_tqp_vector *tqp_vector) 484{ 485 disable_irq(tqp_vector->vector_irq); 486 napi_disable(&tqp_vector->napi); 487 cancel_work_sync(&tqp_vector->rx_group.dim.work); 488 cancel_work_sync(&tqp_vector->tx_group.dim.work); 489} 490 491void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector, 492 u32 rl_value) 493{ 494 u32 rl_reg = hns3_rl_usec_to_reg(rl_value); 495 496 /* this defines the configuration for RL (Interrupt Rate Limiter). 497 * Rl defines rate of interrupts i.e. number of interrupts-per-second 498 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing 499 */ 500 if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable && 501 !tqp_vector->rx_group.coal.adapt_enable) 502 /* According to the hardware, the range of rl_reg is 503 * 0-59 and the unit is 4. 504 */ 505 rl_reg |= HNS3_INT_RL_ENABLE_MASK; 506 507 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET); 508} 509 510void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector, 511 u32 gl_value) 512{ 513 u32 new_val; 514 515 if (tqp_vector->rx_group.coal.unit_1us) 516 new_val = gl_value | HNS3_INT_GL_1US; 517 else 518 new_val = hns3_gl_usec_to_reg(gl_value); 519 520 writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET); 521} 522 523void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector, 524 u32 gl_value) 525{ 526 u32 new_val; 527 528 if (tqp_vector->tx_group.coal.unit_1us) 529 new_val = gl_value | HNS3_INT_GL_1US; 530 else 531 new_val = hns3_gl_usec_to_reg(gl_value); 532 533 writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET); 534} 535 536void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector, 537 u32 ql_value) 538{ 539 writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET); 540} 541 542void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector, 543 u32 ql_value) 544{ 545 writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET); 546} 547 548static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector, 549 struct hns3_nic_priv *priv) 550{ 551 struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal; 552 struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal; 553 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); 554 struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal; 555 struct hns3_enet_coalesce *prx_coal = &priv->rx_coal; 556 557 tx_coal->adapt_enable = ptx_coal->adapt_enable; 558 rx_coal->adapt_enable = prx_coal->adapt_enable; 559 560 tx_coal->int_gl = ptx_coal->int_gl; 561 rx_coal->int_gl = prx_coal->int_gl; 562 563 rx_coal->flow_level = prx_coal->flow_level; 564 tx_coal->flow_level = ptx_coal->flow_level; 565 566 /* device version above V3(include V3), GL can configure 1us 567 * unit, so uses 1us unit. 568 */ 569 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) { 570 tx_coal->unit_1us = 1; 571 rx_coal->unit_1us = 1; 572 } 573 574 if (ae_dev->dev_specs.int_ql_max) { 575 tx_coal->ql_enable = 1; 576 rx_coal->ql_enable = 1; 577 tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max; 578 rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max; 579 tx_coal->int_ql = ptx_coal->int_ql; 580 rx_coal->int_ql = prx_coal->int_ql; 581 } 582} 583 584static void 585hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector, 586 struct hns3_nic_priv *priv) 587{ 588 struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal; 589 struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal; 590 struct hnae3_handle *h = priv->ae_handle; 591 592 hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl); 593 hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl); 594 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting); 595 596 if (tx_coal->ql_enable) 597 hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql); 598 599 if (rx_coal->ql_enable) 600 hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql); 601} 602 603static int hns3_nic_set_real_num_queue(struct net_device *netdev) 604{ 605 struct hnae3_handle *h = hns3_get_handle(netdev); 606 struct hnae3_knic_private_info *kinfo = &h->kinfo; 607 struct hnae3_tc_info *tc_info = &kinfo->tc_info; 608 unsigned int queue_size = kinfo->num_tqps; 609 int i, ret; 610 611 if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) { 612 netdev_reset_tc(netdev); 613 } else { 614 ret = netdev_set_num_tc(netdev, tc_info->num_tc); 615 if (ret) { 616 netdev_err(netdev, 617 "netdev_set_num_tc fail, ret=%d!\n", ret); 618 return ret; 619 } 620 621 for (i = 0; i < tc_info->num_tc; i++) 622 netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i], 623 tc_info->tqp_offset[i]); 624 } 625 626 ret = netif_set_real_num_tx_queues(netdev, queue_size); 627 if (ret) { 628 netdev_err(netdev, 629 "netif_set_real_num_tx_queues fail, ret=%d!\n", ret); 630 return ret; 631 } 632 633 ret = netif_set_real_num_rx_queues(netdev, queue_size); 634 if (ret) { 635 netdev_err(netdev, 636 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret); 637 return ret; 638 } 639 640 return 0; 641} 642 643u16 hns3_get_max_available_channels(struct hnae3_handle *h) 644{ 645 u16 alloc_tqps, max_rss_size, rss_size; 646 647 h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size); 648 rss_size = alloc_tqps / h->kinfo.tc_info.num_tc; 649 650 return min_t(u16, rss_size, max_rss_size); 651} 652 653static void hns3_tqp_enable(struct hnae3_queue *tqp) 654{ 655 u32 rcb_reg; 656 657 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG); 658 rcb_reg |= BIT(HNS3_RING_EN_B); 659 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg); 660} 661 662static void hns3_tqp_disable(struct hnae3_queue *tqp) 663{ 664 u32 rcb_reg; 665 666 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG); 667 rcb_reg &= ~BIT(HNS3_RING_EN_B); 668 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg); 669} 670 671static void hns3_free_rx_cpu_rmap(struct net_device *netdev) 672{ 673#ifdef CONFIG_RFS_ACCEL 674 free_irq_cpu_rmap(netdev->rx_cpu_rmap); 675 netdev->rx_cpu_rmap = NULL; 676#endif 677} 678 679static int hns3_set_rx_cpu_rmap(struct net_device *netdev) 680{ 681#ifdef CONFIG_RFS_ACCEL 682 struct hns3_nic_priv *priv = netdev_priv(netdev); 683 struct hns3_enet_tqp_vector *tqp_vector; 684 int i, ret; 685 686 if (!netdev->rx_cpu_rmap) { 687 netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num); 688 if (!netdev->rx_cpu_rmap) 689 return -ENOMEM; 690 } 691 692 for (i = 0; i < priv->vector_num; i++) { 693 tqp_vector = &priv->tqp_vector[i]; 694 ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap, 695 tqp_vector->vector_irq); 696 if (ret) { 697 hns3_free_rx_cpu_rmap(netdev); 698 return ret; 699 } 700 } 701#endif 702 return 0; 703} 704 705static void hns3_enable_irqs_and_tqps(struct net_device *netdev) 706{ 707 struct hns3_nic_priv *priv = netdev_priv(netdev); 708 struct hnae3_handle *h = priv->ae_handle; 709 u16 i; 710 711 for (i = 0; i < priv->vector_num; i++) 712 hns3_irq_enable(&priv->tqp_vector[i]); 713 714 for (i = 0; i < priv->vector_num; i++) 715 hns3_mask_vector_irq(&priv->tqp_vector[i], 1); 716 717 for (i = 0; i < h->kinfo.num_tqps; i++) 718 hns3_tqp_enable(h->kinfo.tqp[i]); 719} 720 721static void hns3_disable_irqs_and_tqps(struct net_device *netdev) 722{ 723 struct hns3_nic_priv *priv = netdev_priv(netdev); 724 struct hnae3_handle *h = priv->ae_handle; 725 u16 i; 726 727 for (i = 0; i < h->kinfo.num_tqps; i++) 728 hns3_tqp_disable(h->kinfo.tqp[i]); 729 730 for (i = 0; i < priv->vector_num; i++) 731 hns3_mask_vector_irq(&priv->tqp_vector[i], 0); 732 733 for (i = 0; i < priv->vector_num; i++) 734 hns3_irq_disable(&priv->tqp_vector[i]); 735} 736 737static int hns3_nic_net_up(struct net_device *netdev) 738{ 739 struct hns3_nic_priv *priv = netdev_priv(netdev); 740 struct hnae3_handle *h = priv->ae_handle; 741 int ret; 742 743 ret = hns3_nic_reset_all_ring(h); 744 if (ret) 745 return ret; 746 747 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); 748 749 hns3_enable_irqs_and_tqps(netdev); 750 751 /* start the ae_dev */ 752 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0; 753 if (ret) { 754 set_bit(HNS3_NIC_STATE_DOWN, &priv->state); 755 hns3_disable_irqs_and_tqps(netdev); 756 } 757 758 return ret; 759} 760 761static void hns3_config_xps(struct hns3_nic_priv *priv) 762{ 763 int i; 764 765 for (i = 0; i < priv->vector_num; i++) { 766 struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i]; 767 struct hns3_enet_ring *ring = tqp_vector->tx_group.ring; 768 769 while (ring) { 770 int ret; 771 772 ret = netif_set_xps_queue(priv->netdev, 773 &tqp_vector->affinity_mask, 774 ring->tqp->tqp_index); 775 if (ret) 776 netdev_warn(priv->netdev, 777 "set xps queue failed: %d", ret); 778 779 ring = ring->next; 780 } 781 } 782} 783 784static int hns3_nic_net_open(struct net_device *netdev) 785{ 786 struct hns3_nic_priv *priv = netdev_priv(netdev); 787 struct hnae3_handle *h = hns3_get_handle(netdev); 788 struct hnae3_knic_private_info *kinfo; 789 int i, ret; 790 791 if (hns3_nic_resetting(netdev)) 792 return -EBUSY; 793 794 if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) { 795 netdev_warn(netdev, "net open repeatedly!\n"); 796 return 0; 797 } 798 799 netif_carrier_off(netdev); 800 801 ret = hns3_nic_set_real_num_queue(netdev); 802 if (ret) 803 return ret; 804 805 ret = hns3_nic_net_up(netdev); 806 if (ret) { 807 netdev_err(netdev, "net up fail, ret=%d!\n", ret); 808 return ret; 809 } 810 811 kinfo = &h->kinfo; 812 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) 813 netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]); 814 815 if (h->ae_algo->ops->set_timer_task) 816 h->ae_algo->ops->set_timer_task(priv->ae_handle, true); 817 818 hns3_config_xps(priv); 819 820 netif_dbg(h, drv, netdev, "net open\n"); 821 822 return 0; 823} 824 825static void hns3_reset_tx_queue(struct hnae3_handle *h) 826{ 827 struct net_device *ndev = h->kinfo.netdev; 828 struct hns3_nic_priv *priv = netdev_priv(ndev); 829 struct netdev_queue *dev_queue; 830 u32 i; 831 832 for (i = 0; i < h->kinfo.num_tqps; i++) { 833 dev_queue = netdev_get_tx_queue(ndev, 834 priv->ring[i].queue_index); 835 netdev_tx_reset_queue(dev_queue); 836 } 837} 838 839static void hns3_nic_net_down(struct net_device *netdev) 840{ 841 struct hns3_nic_priv *priv = netdev_priv(netdev); 842 const struct hnae3_ae_ops *ops; 843 844 hns3_disable_irqs_and_tqps(netdev); 845 846 /* stop ae_dev */ 847 ops = priv->ae_handle->ae_algo->ops; 848 if (ops->stop) 849 ops->stop(priv->ae_handle); 850 851 /* delay ring buffer clearing to hns3_reset_notify_uninit_enet 852 * during reset process, because driver may not be able 853 * to disable the ring through firmware when downing the netdev. 854 */ 855 if (!hns3_nic_resetting(netdev)) 856 hns3_clear_all_ring(priv->ae_handle, false); 857 858 hns3_reset_tx_queue(priv->ae_handle); 859} 860 861static int hns3_nic_net_stop(struct net_device *netdev) 862{ 863 struct hns3_nic_priv *priv = netdev_priv(netdev); 864 struct hnae3_handle *h = hns3_get_handle(netdev); 865 866 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 867 return 0; 868 869 netif_dbg(h, drv, netdev, "net stop\n"); 870 871 if (h->ae_algo->ops->set_timer_task) 872 h->ae_algo->ops->set_timer_task(priv->ae_handle, false); 873 874 netif_carrier_off(netdev); 875 netif_tx_disable(netdev); 876 877 hns3_nic_net_down(netdev); 878 879 return 0; 880} 881 882static int hns3_nic_uc_sync(struct net_device *netdev, 883 const unsigned char *addr) 884{ 885 struct hnae3_handle *h = hns3_get_handle(netdev); 886 887 if (h->ae_algo->ops->add_uc_addr) 888 return h->ae_algo->ops->add_uc_addr(h, addr); 889 890 return 0; 891} 892 893static int hns3_nic_uc_unsync(struct net_device *netdev, 894 const unsigned char *addr) 895{ 896 struct hnae3_handle *h = hns3_get_handle(netdev); 897 898 /* need ignore the request of removing device address, because 899 * we store the device address and other addresses of uc list 900 * in the function's mac filter list. 901 */ 902 if (ether_addr_equal(addr, netdev->dev_addr)) 903 return 0; 904 905 if (h->ae_algo->ops->rm_uc_addr) 906 return h->ae_algo->ops->rm_uc_addr(h, addr); 907 908 return 0; 909} 910 911static int hns3_nic_mc_sync(struct net_device *netdev, 912 const unsigned char *addr) 913{ 914 struct hnae3_handle *h = hns3_get_handle(netdev); 915 916 if (h->ae_algo->ops->add_mc_addr) 917 return h->ae_algo->ops->add_mc_addr(h, addr); 918 919 return 0; 920} 921 922static int hns3_nic_mc_unsync(struct net_device *netdev, 923 const unsigned char *addr) 924{ 925 struct hnae3_handle *h = hns3_get_handle(netdev); 926 927 if (h->ae_algo->ops->rm_mc_addr) 928 return h->ae_algo->ops->rm_mc_addr(h, addr); 929 930 return 0; 931} 932 933static u8 hns3_get_netdev_flags(struct net_device *netdev) 934{ 935 u8 flags = 0; 936 937 if (netdev->flags & IFF_PROMISC) 938 flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE; 939 else if (netdev->flags & IFF_ALLMULTI) 940 flags = HNAE3_USER_MPE; 941 942 return flags; 943} 944 945static void hns3_nic_set_rx_mode(struct net_device *netdev) 946{ 947 struct hnae3_handle *h = hns3_get_handle(netdev); 948 u8 new_flags; 949 950 new_flags = hns3_get_netdev_flags(netdev); 951 952 __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync); 953 __dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync); 954 955 /* User mode Promisc mode enable and vlan filtering is disabled to 956 * let all packets in. 957 */ 958 h->netdev_flags = new_flags; 959 hns3_request_update_promisc_mode(h); 960} 961 962void hns3_request_update_promisc_mode(struct hnae3_handle *handle) 963{ 964 const struct hnae3_ae_ops *ops = hns3_get_ops(handle); 965 966 if (ops->request_update_promisc_mode) 967 ops->request_update_promisc_mode(handle); 968} 969 970static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring) 971{ 972 struct hns3_tx_spare *tx_spare = ring->tx_spare; 973 u32 ntc, ntu; 974 975 /* This smp_load_acquire() pairs with smp_store_release() in 976 * hns3_tx_spare_update() called in tx desc cleaning process. 977 */ 978 ntc = smp_load_acquire(&tx_spare->last_to_clean); 979 ntu = tx_spare->next_to_use; 980 981 if (ntc > ntu) 982 return ntc - ntu - 1; 983 984 /* The free tx buffer is divided into two part, so pick the 985 * larger one. 986 */ 987 return max(ntc, tx_spare->len - ntu) - 1; 988} 989 990static void hns3_tx_spare_update(struct hns3_enet_ring *ring) 991{ 992 struct hns3_tx_spare *tx_spare = ring->tx_spare; 993 994 if (!tx_spare || 995 tx_spare->last_to_clean == tx_spare->next_to_clean) 996 return; 997 998 /* This smp_store_release() pairs with smp_load_acquire() in 999 * hns3_tx_spare_space() called in xmit process. 1000 */ 1001 smp_store_release(&tx_spare->last_to_clean, 1002 tx_spare->next_to_clean); 1003} 1004 1005static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring, 1006 struct sk_buff *skb, 1007 u32 space) 1008{ 1009 u32 len = skb->len <= ring->tx_copybreak ? skb->len : 1010 skb_headlen(skb); 1011 1012 if (len > ring->tx_copybreak) 1013 return false; 1014 1015 if (ALIGN(len, dma_get_cache_alignment()) > space) { 1016 hns3_ring_stats_update(ring, tx_spare_full); 1017 return false; 1018 } 1019 1020 return true; 1021} 1022 1023static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring, 1024 struct sk_buff *skb, 1025 u32 space) 1026{ 1027 if (skb->len <= ring->tx_copybreak || !tx_sgl || 1028 (!skb_has_frag_list(skb) && 1029 skb_shinfo(skb)->nr_frags < tx_sgl)) 1030 return false; 1031 1032 if (space < HNS3_MAX_SGL_SIZE) { 1033 hns3_ring_stats_update(ring, tx_spare_full); 1034 return false; 1035 } 1036 1037 return true; 1038} 1039 1040static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) 1041{ 1042 u32 alloc_size = ring->tqp->handle->kinfo.tx_spare_buf_size; 1043 struct net_device *netdev = ring_to_netdev(ring); 1044 struct hns3_nic_priv *priv = netdev_priv(netdev); 1045 struct hns3_tx_spare *tx_spare; 1046 struct page *page; 1047 dma_addr_t dma; 1048 int order; 1049 1050 if (!alloc_size) 1051 goto not_init; 1052 1053 order = get_order(alloc_size); 1054 if (order > MAX_PAGE_ORDER) { 1055 if (net_ratelimit()) 1056 dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n"); 1057 goto not_init; 1058 } 1059 1060 tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare), 1061 GFP_KERNEL); 1062 if (!tx_spare) { 1063 /* The driver still work without the tx spare buffer */ 1064 dev_warn(ring_to_dev(ring), "failed to allocate hns3_tx_spare\n"); 1065 goto devm_kzalloc_error; 1066 } 1067 1068 page = alloc_pages_node(dev_to_node(ring_to_dev(ring)), 1069 GFP_KERNEL, order); 1070 if (!page) { 1071 dev_warn(ring_to_dev(ring), "failed to allocate tx spare pages\n"); 1072 goto alloc_pages_error; 1073 } 1074 1075 dma = dma_map_page(ring_to_dev(ring), page, 0, 1076 PAGE_SIZE << order, DMA_TO_DEVICE); 1077 if (dma_mapping_error(ring_to_dev(ring), dma)) { 1078 dev_warn(ring_to_dev(ring), "failed to map pages for tx spare\n"); 1079 goto dma_mapping_error; 1080 } 1081 1082 tx_spare->dma = dma; 1083 tx_spare->buf = page_address(page); 1084 tx_spare->len = PAGE_SIZE << order; 1085 ring->tx_spare = tx_spare; 1086 ring->tx_copybreak = priv->tx_copybreak; 1087 return; 1088 1089dma_mapping_error: 1090 put_page(page); 1091alloc_pages_error: 1092 devm_kfree(ring_to_dev(ring), tx_spare); 1093devm_kzalloc_error: 1094 ring->tqp->handle->kinfo.tx_spare_buf_size = 0; 1095not_init: 1096 /* When driver init or reset_init, the ring->tx_spare is always NULL; 1097 * but when called from hns3_set_ringparam, it's usually not NULL, and 1098 * will be restored if hns3_init_all_ring() failed. So it's safe to set 1099 * ring->tx_spare to NULL here. 1100 */ 1101 ring->tx_spare = NULL; 1102} 1103 1104/* Use hns3_tx_spare_space() to make sure there is enough buffer 1105 * before calling below function to allocate tx buffer. 1106 */ 1107static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring, 1108 unsigned int size, dma_addr_t *dma, 1109 u32 *cb_len) 1110{ 1111 struct hns3_tx_spare *tx_spare = ring->tx_spare; 1112 u32 ntu = tx_spare->next_to_use; 1113 1114 size = ALIGN(size, dma_get_cache_alignment()); 1115 *cb_len = size; 1116 1117 /* Tx spare buffer wraps back here because the end of 1118 * freed tx buffer is not enough. 1119 */ 1120 if (ntu + size > tx_spare->len) { 1121 *cb_len += (tx_spare->len - ntu); 1122 ntu = 0; 1123 } 1124 1125 tx_spare->next_to_use = ntu + size; 1126 if (tx_spare->next_to_use == tx_spare->len) 1127 tx_spare->next_to_use = 0; 1128 1129 *dma = tx_spare->dma + ntu; 1130 1131 return tx_spare->buf + ntu; 1132} 1133 1134static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len) 1135{ 1136 struct hns3_tx_spare *tx_spare = ring->tx_spare; 1137 1138 if (len > tx_spare->next_to_use) { 1139 len -= tx_spare->next_to_use; 1140 tx_spare->next_to_use = tx_spare->len - len; 1141 } else { 1142 tx_spare->next_to_use -= len; 1143 } 1144} 1145 1146static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring, 1147 struct hns3_desc_cb *cb) 1148{ 1149 struct hns3_tx_spare *tx_spare = ring->tx_spare; 1150 u32 ntc = tx_spare->next_to_clean; 1151 u32 len = cb->length; 1152 1153 tx_spare->next_to_clean += len; 1154 1155 if (tx_spare->next_to_clean >= tx_spare->len) { 1156 tx_spare->next_to_clean -= tx_spare->len; 1157 1158 if (tx_spare->next_to_clean) { 1159 ntc = 0; 1160 len = tx_spare->next_to_clean; 1161 } 1162 } 1163 1164 /* This tx spare buffer is only really reclaimed after calling 1165 * hns3_tx_spare_update(), so it is still safe to use the info in 1166 * the tx buffer to do the dma sync or sg unmapping after 1167 * tx_spare->next_to_clean is moved forword. 1168 */ 1169 if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) { 1170 dma_addr_t dma = tx_spare->dma + ntc; 1171 1172 dma_sync_single_for_cpu(ring_to_dev(ring), dma, len, 1173 DMA_TO_DEVICE); 1174 } else { 1175 struct sg_table *sgt = tx_spare->buf + ntc; 1176 1177 dma_unmap_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents, 1178 DMA_TO_DEVICE); 1179 } 1180} 1181 1182static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs, 1183 u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes) 1184{ 1185 u32 l4_offset, hdr_len; 1186 union l3_hdr_info l3; 1187 union l4_hdr_info l4; 1188 u32 l4_paylen; 1189 int ret; 1190 1191 if (!skb_is_gso(skb)) 1192 return 0; 1193 1194 ret = skb_cow_head(skb, 0); 1195 if (unlikely(ret < 0)) 1196 return ret; 1197 1198 l3.hdr = skb_network_header(skb); 1199 l4.hdr = skb_transport_header(skb); 1200 1201 /* Software should clear the IPv4's checksum field when tso is 1202 * needed. 1203 */ 1204 if (l3.v4->version == 4) 1205 l3.v4->check = 0; 1206 1207 /* tunnel packet */ 1208 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 1209 SKB_GSO_GRE_CSUM | 1210 SKB_GSO_UDP_TUNNEL | 1211 SKB_GSO_UDP_TUNNEL_CSUM)) { 1212 /* reset l3&l4 pointers from outer to inner headers */ 1213 l3.hdr = skb_inner_network_header(skb); 1214 l4.hdr = skb_inner_transport_header(skb); 1215 1216 /* Software should clear the IPv4's checksum field when 1217 * tso is needed. 1218 */ 1219 if (l3.v4->version == 4) 1220 l3.v4->check = 0; 1221 } 1222 1223 /* normal or tunnel packet */ 1224 l4_offset = l4.hdr - skb->data; 1225 1226 /* remove payload length from inner pseudo checksum when tso */ 1227 l4_paylen = skb->len - l4_offset; 1228 1229 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 1230 hdr_len = sizeof(*l4.udp) + l4_offset; 1231 csum_replace_by_diff(&l4.udp->check, 1232 (__force __wsum)htonl(l4_paylen)); 1233 } else { 1234 hdr_len = (l4.tcp->doff << 2) + l4_offset; 1235 csum_replace_by_diff(&l4.tcp->check, 1236 (__force __wsum)htonl(l4_paylen)); 1237 } 1238 1239 *send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len; 1240 1241 /* find the txbd field values */ 1242 *paylen_fdop_ol4cs = skb->len - hdr_len; 1243 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1); 1244 1245 /* offload outer UDP header checksum */ 1246 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1247 hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1); 1248 1249 /* get MSS for TSO */ 1250 *mss = skb_shinfo(skb)->gso_size; 1251 1252 trace_hns3_tso(skb); 1253 1254 return 0; 1255} 1256 1257static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto, 1258 u8 *il4_proto) 1259{ 1260 union l3_hdr_info l3; 1261 unsigned char *l4_hdr; 1262 unsigned char *exthdr; 1263 u8 l4_proto_tmp; 1264 __be16 frag_off; 1265 1266 /* find outer header point */ 1267 l3.hdr = skb_network_header(skb); 1268 l4_hdr = skb_transport_header(skb); 1269 1270 if (skb->protocol == htons(ETH_P_IPV6)) { 1271 exthdr = l3.hdr + sizeof(*l3.v6); 1272 l4_proto_tmp = l3.v6->nexthdr; 1273 if (l4_hdr != exthdr) 1274 ipv6_skip_exthdr(skb, exthdr - skb->data, 1275 &l4_proto_tmp, &frag_off); 1276 } else if (skb->protocol == htons(ETH_P_IP)) { 1277 l4_proto_tmp = l3.v4->protocol; 1278 } else { 1279 return -EINVAL; 1280 } 1281 1282 *ol4_proto = l4_proto_tmp; 1283 1284 /* tunnel packet */ 1285 if (!skb->encapsulation) { 1286 *il4_proto = 0; 1287 return 0; 1288 } 1289 1290 /* find inner header point */ 1291 l3.hdr = skb_inner_network_header(skb); 1292 l4_hdr = skb_inner_transport_header(skb); 1293 1294 if (l3.v6->version == 6) { 1295 exthdr = l3.hdr + sizeof(*l3.v6); 1296 l4_proto_tmp = l3.v6->nexthdr; 1297 if (l4_hdr != exthdr) 1298 ipv6_skip_exthdr(skb, exthdr - skb->data, 1299 &l4_proto_tmp, &frag_off); 1300 } else if (l3.v4->version == 4) { 1301 l4_proto_tmp = l3.v4->protocol; 1302 } 1303 1304 *il4_proto = l4_proto_tmp; 1305 1306 return 0; 1307} 1308 1309/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL 1310 * and it is udp packet, which has a dest port as the IANA assigned. 1311 * the hardware is expected to do the checksum offload, but the 1312 * hardware will not do the checksum offload when udp dest port is 1313 * 4789, 4790 or 6081. 1314 */ 1315static bool hns3_tunnel_csum_bug(struct sk_buff *skb) 1316{ 1317 struct hns3_nic_priv *priv = netdev_priv(skb->dev); 1318 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); 1319 union l4_hdr_info l4; 1320 1321 /* device version above V3(include V3), the hardware can 1322 * do this checksum offload. 1323 */ 1324 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) 1325 return false; 1326 1327 l4.hdr = skb_transport_header(skb); 1328 1329 if (!(!skb->encapsulation && 1330 (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) || 1331 l4.udp->dest == htons(GENEVE_UDP_PORT) || 1332 l4.udp->dest == htons(IANA_VXLAN_GPE_UDP_PORT)))) 1333 return false; 1334 1335 return true; 1336} 1337 1338static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto, 1339 u32 *ol_type_vlan_len_msec) 1340{ 1341 u32 l2_len, l3_len, l4_len; 1342 unsigned char *il2_hdr; 1343 union l3_hdr_info l3; 1344 union l4_hdr_info l4; 1345 1346 l3.hdr = skb_network_header(skb); 1347 l4.hdr = skb_transport_header(skb); 1348 1349 /* compute OL2 header size, defined in 2 Bytes */ 1350 l2_len = l3.hdr - skb->data; 1351 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1); 1352 1353 /* compute OL3 header size, defined in 4 Bytes */ 1354 l3_len = l4.hdr - l3.hdr; 1355 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2); 1356 1357 il2_hdr = skb_inner_mac_header(skb); 1358 /* compute OL4 header size, defined in 4 Bytes */ 1359 l4_len = il2_hdr - l4.hdr; 1360 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2); 1361 1362 /* define outer network header type */ 1363 if (skb->protocol == htons(ETH_P_IP)) { 1364 if (skb_is_gso(skb)) 1365 hns3_set_field(*ol_type_vlan_len_msec, 1366 HNS3_TXD_OL3T_S, 1367 HNS3_OL3T_IPV4_CSUM); 1368 else 1369 hns3_set_field(*ol_type_vlan_len_msec, 1370 HNS3_TXD_OL3T_S, 1371 HNS3_OL3T_IPV4_NO_CSUM); 1372 } else if (skb->protocol == htons(ETH_P_IPV6)) { 1373 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S, 1374 HNS3_OL3T_IPV6); 1375 } 1376 1377 if (ol4_proto == IPPROTO_UDP) 1378 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S, 1379 HNS3_TUN_MAC_IN_UDP); 1380 else if (ol4_proto == IPPROTO_GRE) 1381 hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S, 1382 HNS3_TUN_NVGRE); 1383} 1384 1385static void hns3_set_l3_type(struct sk_buff *skb, union l3_hdr_info l3, 1386 u32 *type_cs_vlan_tso) 1387{ 1388 if (l3.v4->version == 4) { 1389 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S, 1390 HNS3_L3T_IPV4); 1391 1392 /* the stack computes the IP header already, the only time we 1393 * need the hardware to recompute it is in the case of TSO. 1394 */ 1395 if (skb_is_gso(skb)) 1396 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1); 1397 } else if (l3.v6->version == 6) { 1398 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S, 1399 HNS3_L3T_IPV6); 1400 } 1401} 1402 1403static int hns3_set_l4_csum_length(struct sk_buff *skb, union l4_hdr_info l4, 1404 u32 l4_proto, u32 *type_cs_vlan_tso) 1405{ 1406 /* compute inner(/normal) L4 header size, defined in 4 Bytes */ 1407 switch (l4_proto) { 1408 case IPPROTO_TCP: 1409 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); 1410 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S, 1411 HNS3_L4T_TCP); 1412 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S, 1413 l4.tcp->doff); 1414 break; 1415 case IPPROTO_UDP: 1416 if (hns3_tunnel_csum_bug(skb)) { 1417 int ret = skb_put_padto(skb, HNS3_MIN_TUN_PKT_LEN); 1418 1419 return ret ? ret : skb_checksum_help(skb); 1420 } 1421 1422 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); 1423 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S, 1424 HNS3_L4T_UDP); 1425 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S, 1426 (sizeof(struct udphdr) >> 2)); 1427 break; 1428 case IPPROTO_SCTP: 1429 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1); 1430 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S, 1431 HNS3_L4T_SCTP); 1432 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S, 1433 (sizeof(struct sctphdr) >> 2)); 1434 break; 1435 default: 1436 /* drop the skb tunnel packet if hardware don't support, 1437 * because hardware can't calculate csum when TSO. 1438 */ 1439 if (skb_is_gso(skb)) 1440 return -EDOM; 1441 1442 /* the stack computes the IP header already, 1443 * driver calculate l4 checksum when not TSO. 1444 */ 1445 return skb_checksum_help(skb); 1446 } 1447 1448 return 0; 1449} 1450 1451static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, 1452 u8 il4_proto, u32 *type_cs_vlan_tso, 1453 u32 *ol_type_vlan_len_msec) 1454{ 1455 unsigned char *l2_hdr = skb->data; 1456 u32 l4_proto = ol4_proto; 1457 union l4_hdr_info l4; 1458 union l3_hdr_info l3; 1459 u32 l2_len, l3_len; 1460 1461 l4.hdr = skb_transport_header(skb); 1462 l3.hdr = skb_network_header(skb); 1463 1464 /* handle encapsulation skb */ 1465 if (skb->encapsulation) { 1466 /* If this is a not UDP/GRE encapsulation skb */ 1467 if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) { 1468 /* drop the skb tunnel packet if hardware don't support, 1469 * because hardware can't calculate csum when TSO. 1470 */ 1471 if (skb_is_gso(skb)) 1472 return -EDOM; 1473 1474 /* the stack computes the IP header already, 1475 * driver calculate l4 checksum when not TSO. 1476 */ 1477 return skb_checksum_help(skb); 1478 } 1479 1480 hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec); 1481 1482 /* switch to inner header */ 1483 l2_hdr = skb_inner_mac_header(skb); 1484 l3.hdr = skb_inner_network_header(skb); 1485 l4.hdr = skb_inner_transport_header(skb); 1486 l4_proto = il4_proto; 1487 } 1488 1489 hns3_set_l3_type(skb, l3, type_cs_vlan_tso); 1490 1491 /* compute inner(/normal) L2 header size, defined in 2 Bytes */ 1492 l2_len = l3.hdr - l2_hdr; 1493 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1); 1494 1495 /* compute inner(/normal) L3 header size, defined in 4 Bytes */ 1496 l3_len = l4.hdr - l3.hdr; 1497 hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2); 1498 1499 return hns3_set_l4_csum_length(skb, l4, l4_proto, type_cs_vlan_tso); 1500} 1501 1502static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring, 1503 struct sk_buff *skb) 1504{ 1505 struct hnae3_handle *handle = tx_ring->tqp->handle; 1506 struct hnae3_ae_dev *ae_dev; 1507 struct vlan_ethhdr *vhdr; 1508 int rc; 1509 1510 if (!(skb->protocol == htons(ETH_P_8021Q) || 1511 skb_vlan_tag_present(skb))) 1512 return 0; 1513 1514 /* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert 1515 * VLAN enabled, only one VLAN header is allowed in skb, otherwise it 1516 * will cause RAS error. 1517 */ 1518 ae_dev = hns3_get_ae_dev(handle); 1519 if (unlikely(skb_vlan_tagged_multi(skb) && 1520 ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 && 1521 handle->port_base_vlan_state == 1522 HNAE3_PORT_BASE_VLAN_ENABLE)) 1523 return -EINVAL; 1524 1525 if (skb->protocol == htons(ETH_P_8021Q) && 1526 !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 1527 /* When HW VLAN acceleration is turned off, and the stack 1528 * sets the protocol to 802.1q, the driver just need to 1529 * set the protocol to the encapsulated ethertype. 1530 */ 1531 skb->protocol = vlan_get_protocol(skb); 1532 return 0; 1533 } 1534 1535 if (skb_vlan_tag_present(skb)) { 1536 /* Based on hw strategy, use out_vtag in two layer tag case, 1537 * and use inner_vtag in one tag case. 1538 */ 1539 if (skb->protocol == htons(ETH_P_8021Q) && 1540 handle->port_base_vlan_state == 1541 HNAE3_PORT_BASE_VLAN_DISABLE) 1542 rc = HNS3_OUTER_VLAN_TAG; 1543 else 1544 rc = HNS3_INNER_VLAN_TAG; 1545 1546 skb->protocol = vlan_get_protocol(skb); 1547 return rc; 1548 } 1549 1550 rc = skb_cow_head(skb, 0); 1551 if (unlikely(rc < 0)) 1552 return rc; 1553 1554 vhdr = skb_vlan_eth_hdr(skb); 1555 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT) 1556 & VLAN_PRIO_MASK); 1557 1558 skb->protocol = vlan_get_protocol(skb); 1559 return 0; 1560} 1561 1562/* check if the hardware is capable of checksum offloading */ 1563static bool hns3_check_hw_tx_csum(struct sk_buff *skb) 1564{ 1565 struct hns3_nic_priv *priv = netdev_priv(skb->dev); 1566 1567 /* Kindly note, due to backward compatibility of the TX descriptor, 1568 * HW checksum of the non-IP packets and GSO packets is handled at 1569 * different place in the following code 1570 */ 1571 if (skb_csum_is_sctp(skb) || skb_is_gso(skb) || 1572 !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state)) 1573 return false; 1574 1575 return true; 1576} 1577 1578struct hns3_desc_param { 1579 u32 paylen_ol4cs; 1580 u32 ol_type_vlan_len_msec; 1581 u32 type_cs_vlan_tso; 1582 u16 mss_hw_csum; 1583 u16 inner_vtag; 1584 u16 out_vtag; 1585}; 1586 1587static void hns3_init_desc_data(struct sk_buff *skb, struct hns3_desc_param *pa) 1588{ 1589 pa->paylen_ol4cs = skb->len; 1590 pa->ol_type_vlan_len_msec = 0; 1591 pa->type_cs_vlan_tso = 0; 1592 pa->mss_hw_csum = 0; 1593 pa->inner_vtag = 0; 1594 pa->out_vtag = 0; 1595} 1596 1597static int hns3_handle_vlan_info(struct hns3_enet_ring *ring, 1598 struct sk_buff *skb, 1599 struct hns3_desc_param *param) 1600{ 1601 int ret; 1602 1603 ret = hns3_handle_vtags(ring, skb); 1604 if (unlikely(ret < 0)) { 1605 hns3_ring_stats_update(ring, tx_vlan_err); 1606 return ret; 1607 } else if (ret == HNS3_INNER_VLAN_TAG) { 1608 param->inner_vtag = skb_vlan_tag_get(skb); 1609 param->inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & 1610 VLAN_PRIO_MASK; 1611 hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1); 1612 } else if (ret == HNS3_OUTER_VLAN_TAG) { 1613 param->out_vtag = skb_vlan_tag_get(skb); 1614 param->out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & 1615 VLAN_PRIO_MASK; 1616 hns3_set_field(param->ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B, 1617 1); 1618 } 1619 return 0; 1620} 1621 1622static int hns3_handle_csum_partial(struct hns3_enet_ring *ring, 1623 struct sk_buff *skb, 1624 struct hns3_desc_cb *desc_cb, 1625 struct hns3_desc_param *param) 1626{ 1627 u8 ol4_proto, il4_proto; 1628 int ret; 1629 1630 if (hns3_check_hw_tx_csum(skb)) { 1631 /* set checksum start and offset, defined in 2 Bytes */ 1632 hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_CSUM_START_S, 1633 skb_checksum_start_offset(skb) >> 1); 1634 hns3_set_field(param->ol_type_vlan_len_msec, 1635 HNS3_TXD_CSUM_OFFSET_S, 1636 skb->csum_offset >> 1); 1637 param->mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B); 1638 return 0; 1639 } 1640 1641 skb_reset_mac_len(skb); 1642 1643 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); 1644 if (unlikely(ret < 0)) { 1645 hns3_ring_stats_update(ring, tx_l4_proto_err); 1646 return ret; 1647 } 1648 1649 ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto, 1650 ¶m->type_cs_vlan_tso, 1651 ¶m->ol_type_vlan_len_msec); 1652 if (unlikely(ret < 0)) { 1653 hns3_ring_stats_update(ring, tx_l2l3l4_err); 1654 return ret; 1655 } 1656 1657 ret = hns3_set_tso(skb, ¶m->paylen_ol4cs, ¶m->mss_hw_csum, 1658 ¶m->type_cs_vlan_tso, &desc_cb->send_bytes); 1659 if (unlikely(ret < 0)) { 1660 hns3_ring_stats_update(ring, tx_tso_err); 1661 return ret; 1662 } 1663 return 0; 1664} 1665 1666static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, 1667 struct sk_buff *skb, struct hns3_desc *desc, 1668 struct hns3_desc_cb *desc_cb) 1669{ 1670 struct hns3_desc_param param; 1671 int ret; 1672 1673 hns3_init_desc_data(skb, ¶m); 1674 ret = hns3_handle_vlan_info(ring, skb, ¶m); 1675 if (unlikely(ret < 0)) 1676 return ret; 1677 1678 desc_cb->send_bytes = skb->len; 1679 1680 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1681 ret = hns3_handle_csum_partial(ring, skb, desc_cb, ¶m); 1682 if (ret) 1683 return ret; 1684 } 1685 1686 /* Set txbd */ 1687 desc->tx.ol_type_vlan_len_msec = 1688 cpu_to_le32(param.ol_type_vlan_len_msec); 1689 desc->tx.type_cs_vlan_tso_len = cpu_to_le32(param.type_cs_vlan_tso); 1690 desc->tx.paylen_ol4cs = cpu_to_le32(param.paylen_ol4cs); 1691 desc->tx.mss_hw_csum = cpu_to_le16(param.mss_hw_csum); 1692 desc->tx.vlan_tag = cpu_to_le16(param.inner_vtag); 1693 desc->tx.outer_vlan_tag = cpu_to_le16(param.out_vtag); 1694 1695 return 0; 1696} 1697 1698static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma, 1699 unsigned int size) 1700{ 1701#define HNS3_LIKELY_BD_NUM 1 1702 1703 struct hns3_desc *desc = &ring->desc[ring->next_to_use]; 1704 unsigned int frag_buf_num, k; 1705 int sizeoflast; 1706 1707 if (likely(size <= HNS3_MAX_BD_SIZE)) { 1708 desc->addr = cpu_to_le64(dma); 1709 desc->tx.send_size = cpu_to_le16(size); 1710 desc->tx.bdtp_fe_sc_vld_ra_ri = 1711 cpu_to_le16(BIT(HNS3_TXD_VLD_B)); 1712 1713 trace_hns3_tx_desc(ring, ring->next_to_use); 1714 ring_ptr_move_fw(ring, next_to_use); 1715 return HNS3_LIKELY_BD_NUM; 1716 } 1717 1718 frag_buf_num = hns3_tx_bd_count(size); 1719 sizeoflast = size % HNS3_MAX_BD_SIZE; 1720 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE; 1721 1722 /* When frag size is bigger than hardware limit, split this frag */ 1723 for (k = 0; k < frag_buf_num; k++) { 1724 /* now, fill the descriptor */ 1725 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k); 1726 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ? 1727 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE); 1728 desc->tx.bdtp_fe_sc_vld_ra_ri = 1729 cpu_to_le16(BIT(HNS3_TXD_VLD_B)); 1730 1731 trace_hns3_tx_desc(ring, ring->next_to_use); 1732 /* move ring pointer to next */ 1733 ring_ptr_move_fw(ring, next_to_use); 1734 1735 desc = &ring->desc[ring->next_to_use]; 1736 } 1737 1738 return frag_buf_num; 1739} 1740 1741static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv, 1742 unsigned int type) 1743{ 1744 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 1745 struct device *dev = ring_to_dev(ring); 1746 unsigned int size; 1747 dma_addr_t dma; 1748 1749 if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) { 1750 struct sk_buff *skb = (struct sk_buff *)priv; 1751 1752 size = skb_headlen(skb); 1753 if (!size) 1754 return 0; 1755 1756 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); 1757 } else if (type & DESC_TYPE_BOUNCE_HEAD) { 1758 /* Head data has been filled in hns3_handle_tx_bounce(), 1759 * just return 0 here. 1760 */ 1761 return 0; 1762 } else { 1763 skb_frag_t *frag = (skb_frag_t *)priv; 1764 1765 size = skb_frag_size(frag); 1766 if (!size) 1767 return 0; 1768 1769 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); 1770 } 1771 1772 if (unlikely(dma_mapping_error(dev, dma))) { 1773 hns3_ring_stats_update(ring, sw_err_cnt); 1774 return -ENOMEM; 1775 } 1776 1777 desc_cb->priv = priv; 1778 desc_cb->length = size; 1779 desc_cb->dma = dma; 1780 desc_cb->type = type; 1781 1782 return hns3_fill_desc(ring, dma, size); 1783} 1784 1785static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size, 1786 unsigned int bd_num) 1787{ 1788 unsigned int size; 1789 int i; 1790 1791 size = skb_headlen(skb); 1792 while (size > HNS3_MAX_BD_SIZE) { 1793 bd_size[bd_num++] = HNS3_MAX_BD_SIZE; 1794 size -= HNS3_MAX_BD_SIZE; 1795 1796 if (bd_num > HNS3_MAX_TSO_BD_NUM) 1797 return bd_num; 1798 } 1799 1800 if (size) { 1801 bd_size[bd_num++] = size; 1802 if (bd_num > HNS3_MAX_TSO_BD_NUM) 1803 return bd_num; 1804 } 1805 1806 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1807 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1808 size = skb_frag_size(frag); 1809 if (!size) 1810 continue; 1811 1812 while (size > HNS3_MAX_BD_SIZE) { 1813 bd_size[bd_num++] = HNS3_MAX_BD_SIZE; 1814 size -= HNS3_MAX_BD_SIZE; 1815 1816 if (bd_num > HNS3_MAX_TSO_BD_NUM) 1817 return bd_num; 1818 } 1819 1820 bd_size[bd_num++] = size; 1821 if (bd_num > HNS3_MAX_TSO_BD_NUM) 1822 return bd_num; 1823 } 1824 1825 return bd_num; 1826} 1827 1828static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size, 1829 u8 max_non_tso_bd_num, unsigned int bd_num, 1830 unsigned int recursion_level) 1831{ 1832#define HNS3_MAX_RECURSION_LEVEL 24 1833 1834 struct sk_buff *frag_skb; 1835 1836 /* If the total len is within the max bd limit */ 1837 if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level && 1838 !skb_has_frag_list(skb) && 1839 skb_shinfo(skb)->nr_frags < max_non_tso_bd_num)) 1840 return skb_shinfo(skb)->nr_frags + 1U; 1841 1842 if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL)) 1843 return UINT_MAX; 1844 1845 bd_num = hns3_skb_bd_num(skb, bd_size, bd_num); 1846 if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM) 1847 return bd_num; 1848 1849 skb_walk_frags(skb, frag_skb) { 1850 bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num, 1851 bd_num, recursion_level + 1); 1852 if (bd_num > HNS3_MAX_TSO_BD_NUM) 1853 return bd_num; 1854 } 1855 1856 return bd_num; 1857} 1858 1859static unsigned int hns3_gso_hdr_len(struct sk_buff *skb) 1860{ 1861 if (!skb->encapsulation) 1862 return skb_tcp_all_headers(skb); 1863 1864 return skb_inner_tcp_all_headers(skb); 1865} 1866 1867/* HW need every continuous max_non_tso_bd_num buffer data to be larger 1868 * than MSS, we simplify it by ensuring skb_headlen + the first continuous 1869 * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss, 1870 * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger 1871 * than MSS except the last max_non_tso_bd_num - 1 frags. 1872 */ 1873static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size, 1874 unsigned int bd_num, u8 max_non_tso_bd_num) 1875{ 1876 unsigned int tot_len = 0; 1877 unsigned int i; 1878 1879 for (i = 0; i < max_non_tso_bd_num - 1U; i++) 1880 tot_len += bd_size[i]; 1881 1882 /* ensure the first max_non_tso_bd_num frags is greater than 1883 * mss + header 1884 */ 1885 if (tot_len + bd_size[max_non_tso_bd_num - 1U] < 1886 skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb)) 1887 return true; 1888 1889 /* ensure every continuous max_non_tso_bd_num - 1 buffer is greater 1890 * than mss except the last one. 1891 */ 1892 for (i = 0; i < bd_num - max_non_tso_bd_num; i++) { 1893 tot_len -= bd_size[i]; 1894 tot_len += bd_size[i + max_non_tso_bd_num - 1U]; 1895 1896 if (tot_len < skb_shinfo(skb)->gso_size) 1897 return true; 1898 } 1899 1900 return false; 1901} 1902 1903void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size) 1904{ 1905 u32 i; 1906 1907 for (i = 0; i < MAX_SKB_FRAGS; i++) 1908 size[i] = skb_frag_size(&shinfo->frags[i]); 1909} 1910 1911static int hns3_skb_linearize(struct hns3_enet_ring *ring, 1912 struct sk_buff *skb, 1913 unsigned int bd_num) 1914{ 1915 /* 'bd_num == UINT_MAX' means the skb' fraglist has a 1916 * recursion level of over HNS3_MAX_RECURSION_LEVEL. 1917 */ 1918 if (bd_num == UINT_MAX) { 1919 hns3_ring_stats_update(ring, over_max_recursion); 1920 return -ENOMEM; 1921 } 1922 1923 /* The skb->len has exceeded the hw limitation, linearization 1924 * will not help. 1925 */ 1926 if (skb->len > HNS3_MAX_TSO_SIZE || 1927 (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE)) { 1928 hns3_ring_stats_update(ring, hw_limitation); 1929 return -ENOMEM; 1930 } 1931 1932 if (__skb_linearize(skb)) { 1933 hns3_ring_stats_update(ring, sw_err_cnt); 1934 return -ENOMEM; 1935 } 1936 1937 return 0; 1938} 1939 1940static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, 1941 struct net_device *netdev, 1942 struct sk_buff *skb) 1943{ 1944 struct hns3_nic_priv *priv = netdev_priv(netdev); 1945 u8 max_non_tso_bd_num = priv->max_non_tso_bd_num; 1946 unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U]; 1947 unsigned int bd_num; 1948 1949 bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0); 1950 if (unlikely(bd_num > max_non_tso_bd_num)) { 1951 if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) && 1952 !hns3_skb_need_linearized(skb, bd_size, bd_num, 1953 max_non_tso_bd_num)) { 1954 trace_hns3_over_max_bd(skb); 1955 goto out; 1956 } 1957 1958 if (hns3_skb_linearize(ring, skb, bd_num)) 1959 return -ENOMEM; 1960 1961 bd_num = hns3_tx_bd_count(skb->len); 1962 1963 hns3_ring_stats_update(ring, tx_copy); 1964 } 1965 1966out: 1967 if (likely(ring_space(ring) >= bd_num)) 1968 return bd_num; 1969 1970 netif_stop_subqueue(netdev, ring->queue_index); 1971 smp_mb(); /* Memory barrier before checking ring_space */ 1972 1973 /* Start queue in case hns3_clean_tx_ring has just made room 1974 * available and has not seen the queue stopped state performed 1975 * by netif_stop_subqueue above. 1976 */ 1977 if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) && 1978 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) { 1979 netif_start_subqueue(netdev, ring->queue_index); 1980 return bd_num; 1981 } 1982 1983 hns3_ring_stats_update(ring, tx_busy); 1984 1985 return -EBUSY; 1986} 1987 1988static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig) 1989{ 1990 struct device *dev = ring_to_dev(ring); 1991 unsigned int i; 1992 1993 for (i = 0; i < ring->desc_num; i++) { 1994 struct hns3_desc *desc = &ring->desc[ring->next_to_use]; 1995 struct hns3_desc_cb *desc_cb; 1996 1997 memset(desc, 0, sizeof(*desc)); 1998 1999 /* check if this is where we started */ 2000 if (ring->next_to_use == next_to_use_orig) 2001 break; 2002 2003 /* rollback one */ 2004 ring_ptr_move_bw(ring, next_to_use); 2005 2006 desc_cb = &ring->desc_cb[ring->next_to_use]; 2007 2008 if (!desc_cb->dma) 2009 continue; 2010 2011 /* unmap the descriptor dma address */ 2012 if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB)) 2013 dma_unmap_single(dev, desc_cb->dma, desc_cb->length, 2014 DMA_TO_DEVICE); 2015 else if (desc_cb->type & 2016 (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) 2017 hns3_tx_spare_rollback(ring, desc_cb->length); 2018 else if (desc_cb->length) 2019 dma_unmap_page(dev, desc_cb->dma, desc_cb->length, 2020 DMA_TO_DEVICE); 2021 2022 desc_cb->length = 0; 2023 desc_cb->dma = 0; 2024 desc_cb->type = DESC_TYPE_UNKNOWN; 2025 } 2026} 2027 2028static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring, 2029 struct sk_buff *skb, unsigned int type) 2030{ 2031 struct sk_buff *frag_skb; 2032 int i, ret, bd_num = 0; 2033 2034 ret = hns3_map_and_fill_desc(ring, skb, type); 2035 if (unlikely(ret < 0)) 2036 return ret; 2037 2038 bd_num += ret; 2039 2040 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2041 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2042 2043 ret = hns3_map_and_fill_desc(ring, frag, DESC_TYPE_PAGE); 2044 if (unlikely(ret < 0)) 2045 return ret; 2046 2047 bd_num += ret; 2048 } 2049 2050 skb_walk_frags(skb, frag_skb) { 2051 ret = hns3_fill_skb_to_desc(ring, frag_skb, 2052 DESC_TYPE_FRAGLIST_SKB); 2053 if (unlikely(ret < 0)) 2054 return ret; 2055 2056 bd_num += ret; 2057 } 2058 2059 return bd_num; 2060} 2061 2062static void hns3_tx_push_bd(struct hns3_enet_ring *ring, int num) 2063{ 2064#define HNS3_BYTES_PER_64BIT 8 2065 2066 struct hns3_desc desc[HNS3_MAX_PUSH_BD_NUM] = {}; 2067 int offset = 0; 2068 2069 /* make sure everything is visible to device before 2070 * excuting tx push or updating doorbell 2071 */ 2072 dma_wmb(); 2073 2074 do { 2075 int idx = (ring->next_to_use - num + ring->desc_num) % 2076 ring->desc_num; 2077 2078 u64_stats_update_begin(&ring->syncp); 2079 ring->stats.tx_push++; 2080 u64_stats_update_end(&ring->syncp); 2081 memcpy(&desc[offset], &ring->desc[idx], 2082 sizeof(struct hns3_desc)); 2083 offset++; 2084 } while (--num); 2085 2086 __iowrite64_copy(ring->tqp->mem_base, desc, 2087 (sizeof(struct hns3_desc) * HNS3_MAX_PUSH_BD_NUM) / 2088 HNS3_BYTES_PER_64BIT); 2089} 2090 2091static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring) 2092{ 2093#define HNS3_MEM_DOORBELL_OFFSET 64 2094 2095 __le64 bd_num = cpu_to_le64((u64)ring->pending_buf); 2096 2097 /* make sure everything is visible to device before 2098 * excuting tx push or updating doorbell 2099 */ 2100 dma_wmb(); 2101 2102 __iowrite64_copy(ring->tqp->mem_base + HNS3_MEM_DOORBELL_OFFSET, 2103 &bd_num, 1); 2104 u64_stats_update_begin(&ring->syncp); 2105 ring->stats.tx_mem_doorbell += ring->pending_buf; 2106 u64_stats_update_end(&ring->syncp); 2107} 2108 2109static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num, 2110 bool doorbell) 2111{ 2112 struct net_device *netdev = ring_to_netdev(ring); 2113 struct hns3_nic_priv *priv = netdev_priv(netdev); 2114 2115 /* when tx push is enabled, the packet whose number of BD below 2116 * HNS3_MAX_PUSH_BD_NUM can be pushed directly. 2117 */ 2118 if (test_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state) && num && 2119 !ring->pending_buf && num <= HNS3_MAX_PUSH_BD_NUM && doorbell) { 2120 /* This smp_store_release() pairs with smp_load_acquire() in 2121 * hns3_nic_reclaim_desc(). Ensure that the BD valid bit 2122 * is updated. 2123 */ 2124 smp_store_release(&ring->last_to_use, ring->next_to_use); 2125 hns3_tx_push_bd(ring, num); 2126 return; 2127 } 2128 2129 ring->pending_buf += num; 2130 2131 if (!doorbell) { 2132 hns3_ring_stats_update(ring, tx_more); 2133 return; 2134 } 2135 2136 /* This smp_store_release() pairs with smp_load_acquire() in 2137 * hns3_nic_reclaim_desc(). Ensure that the BD valid bit is updated. 2138 */ 2139 smp_store_release(&ring->last_to_use, ring->next_to_use); 2140 2141 if (ring->tqp->mem_base) 2142 hns3_tx_mem_doorbell(ring); 2143 else 2144 writel(ring->pending_buf, 2145 ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG); 2146 2147 ring->pending_buf = 0; 2148} 2149 2150static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb, 2151 struct hns3_desc *desc) 2152{ 2153 struct hnae3_handle *h = hns3_get_handle(netdev); 2154 2155 if (!(h->ae_algo->ops->set_tx_hwts_info && 2156 h->ae_algo->ops->set_tx_hwts_info(h, skb))) 2157 return; 2158 2159 desc->tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_TSYN_B)); 2160} 2161 2162static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring, 2163 struct sk_buff *skb) 2164{ 2165 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 2166 unsigned int type = DESC_TYPE_BOUNCE_HEAD; 2167 unsigned int size = skb_headlen(skb); 2168 dma_addr_t dma; 2169 int bd_num = 0; 2170 u32 cb_len; 2171 void *buf; 2172 int ret; 2173 2174 if (skb->len <= ring->tx_copybreak) { 2175 size = skb->len; 2176 type = DESC_TYPE_BOUNCE_ALL; 2177 } 2178 2179 /* hns3_can_use_tx_bounce() is called to ensure the below 2180 * function can always return the tx buffer. 2181 */ 2182 buf = hns3_tx_spare_alloc(ring, size, &dma, &cb_len); 2183 2184 ret = skb_copy_bits(skb, 0, buf, size); 2185 if (unlikely(ret < 0)) { 2186 hns3_tx_spare_rollback(ring, cb_len); 2187 hns3_ring_stats_update(ring, copy_bits_err); 2188 return ret; 2189 } 2190 2191 desc_cb->priv = skb; 2192 desc_cb->length = cb_len; 2193 desc_cb->dma = dma; 2194 desc_cb->type = type; 2195 2196 bd_num += hns3_fill_desc(ring, dma, size); 2197 2198 if (type == DESC_TYPE_BOUNCE_HEAD) { 2199 ret = hns3_fill_skb_to_desc(ring, skb, 2200 DESC_TYPE_BOUNCE_HEAD); 2201 if (unlikely(ret < 0)) 2202 return ret; 2203 2204 bd_num += ret; 2205 } 2206 2207 dma_sync_single_for_device(ring_to_dev(ring), dma, size, 2208 DMA_TO_DEVICE); 2209 2210 hns3_ring_stats_update(ring, tx_bounce); 2211 2212 return bd_num; 2213} 2214 2215static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring, 2216 struct sk_buff *skb) 2217{ 2218 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 2219 u32 nfrag = skb_shinfo(skb)->nr_frags + 1; 2220 struct sg_table *sgt; 2221 int bd_num = 0; 2222 dma_addr_t dma; 2223 u32 cb_len, i; 2224 int nents; 2225 2226 if (skb_has_frag_list(skb)) 2227 nfrag = HNS3_MAX_TSO_BD_NUM; 2228 2229 /* hns3_can_use_tx_sgl() is called to ensure the below 2230 * function can always return the tx buffer. 2231 */ 2232 sgt = hns3_tx_spare_alloc(ring, HNS3_SGL_SIZE(nfrag), 2233 &dma, &cb_len); 2234 2235 /* scatterlist follows by the sg table */ 2236 sgt->sgl = (struct scatterlist *)(sgt + 1); 2237 sg_init_table(sgt->sgl, nfrag); 2238 nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len); 2239 if (unlikely(nents < 0)) { 2240 hns3_tx_spare_rollback(ring, cb_len); 2241 hns3_ring_stats_update(ring, skb2sgl_err); 2242 return -ENOMEM; 2243 } 2244 2245 sgt->orig_nents = nents; 2246 sgt->nents = dma_map_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents, 2247 DMA_TO_DEVICE); 2248 if (unlikely(!sgt->nents)) { 2249 hns3_tx_spare_rollback(ring, cb_len); 2250 hns3_ring_stats_update(ring, map_sg_err); 2251 return -ENOMEM; 2252 } 2253 2254 desc_cb->priv = skb; 2255 desc_cb->length = cb_len; 2256 desc_cb->dma = dma; 2257 desc_cb->type = DESC_TYPE_SGL_SKB; 2258 2259 for (i = 0; i < sgt->nents; i++) 2260 bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i), 2261 sg_dma_len(sgt->sgl + i)); 2262 hns3_ring_stats_update(ring, tx_sgl); 2263 2264 return bd_num; 2265} 2266 2267static int hns3_handle_desc_filling(struct hns3_enet_ring *ring, 2268 struct sk_buff *skb) 2269{ 2270 u32 space; 2271 2272 if (!ring->tx_spare) 2273 goto out; 2274 2275 space = hns3_tx_spare_space(ring); 2276 2277 if (hns3_can_use_tx_sgl(ring, skb, space)) 2278 return hns3_handle_tx_sgl(ring, skb); 2279 2280 if (hns3_can_use_tx_bounce(ring, skb, space)) 2281 return hns3_handle_tx_bounce(ring, skb); 2282 2283out: 2284 return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); 2285} 2286 2287static int hns3_handle_skb_desc(struct hns3_enet_ring *ring, 2288 struct sk_buff *skb, 2289 struct hns3_desc_cb *desc_cb, 2290 int next_to_use_head) 2291{ 2292 int ret; 2293 2294 ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use], 2295 desc_cb); 2296 if (unlikely(ret < 0)) 2297 goto fill_err; 2298 2299 /* 'ret < 0' means filling error, 'ret == 0' means skb->len is 2300 * zero, which is unlikely, and 'ret > 0' means how many tx desc 2301 * need to be notified to the hw. 2302 */ 2303 ret = hns3_handle_desc_filling(ring, skb); 2304 if (likely(ret > 0)) 2305 return ret; 2306 2307fill_err: 2308 hns3_clear_desc(ring, next_to_use_head); 2309 return ret; 2310} 2311 2312netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) 2313{ 2314 struct hns3_nic_priv *priv = netdev_priv(netdev); 2315 struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping]; 2316 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; 2317 struct netdev_queue *dev_queue; 2318 int pre_ntu, ret; 2319 bool doorbell; 2320 2321 /* Hardware can only handle short frames above 32 bytes */ 2322 if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) { 2323 hns3_tx_doorbell(ring, 0, !netdev_xmit_more()); 2324 2325 hns3_ring_stats_update(ring, sw_err_cnt); 2326 2327 return NETDEV_TX_OK; 2328 } 2329 2330 /* Prefetch the data used later */ 2331 prefetch(skb->data); 2332 2333 ret = hns3_nic_maybe_stop_tx(ring, netdev, skb); 2334 if (unlikely(ret <= 0)) { 2335 if (ret == -EBUSY) { 2336 hns3_tx_doorbell(ring, 0, true); 2337 return NETDEV_TX_BUSY; 2338 } 2339 2340 hns3_rl_err(netdev, "xmit error: %d!\n", ret); 2341 goto out_err_tx_ok; 2342 } 2343 2344 ret = hns3_handle_skb_desc(ring, skb, desc_cb, ring->next_to_use); 2345 if (unlikely(ret <= 0)) 2346 goto out_err_tx_ok; 2347 2348 pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) : 2349 (ring->desc_num - 1); 2350 2351 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 2352 hns3_tsyn(netdev, skb, &ring->desc[pre_ntu]); 2353 2354 ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |= 2355 cpu_to_le16(BIT(HNS3_TXD_FE_B)); 2356 trace_hns3_tx_desc(ring, pre_ntu); 2357 2358 skb_tx_timestamp(skb); 2359 2360 /* Complete translate all packets */ 2361 dev_queue = netdev_get_tx_queue(netdev, ring->queue_index); 2362 doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes, 2363 netdev_xmit_more()); 2364 hns3_tx_doorbell(ring, ret, doorbell); 2365 2366 return NETDEV_TX_OK; 2367 2368out_err_tx_ok: 2369 dev_kfree_skb_any(skb); 2370 hns3_tx_doorbell(ring, 0, !netdev_xmit_more()); 2371 return NETDEV_TX_OK; 2372} 2373 2374static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) 2375{ 2376 char format_mac_addr_perm[HNAE3_FORMAT_MAC_ADDR_LEN]; 2377 char format_mac_addr_sa[HNAE3_FORMAT_MAC_ADDR_LEN]; 2378 struct hnae3_handle *h = hns3_get_handle(netdev); 2379 struct sockaddr *mac_addr = p; 2380 int ret; 2381 2382 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data)) 2383 return -EADDRNOTAVAIL; 2384 2385 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) { 2386 hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data); 2387 netdev_info(netdev, "already using mac address %s\n", 2388 format_mac_addr_sa); 2389 return 0; 2390 } 2391 2392 /* For VF device, if there is a perm_addr, then the user will not 2393 * be allowed to change the address. 2394 */ 2395 if (!hns3_is_phys_func(h->pdev) && 2396 !is_zero_ether_addr(netdev->perm_addr)) { 2397 hnae3_format_mac_addr(format_mac_addr_perm, netdev->perm_addr); 2398 hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data); 2399 netdev_err(netdev, "has permanent MAC %s, user MAC %s not allow\n", 2400 format_mac_addr_perm, format_mac_addr_sa); 2401 return -EPERM; 2402 } 2403 2404 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false); 2405 if (ret) { 2406 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret); 2407 return ret; 2408 } 2409 2410 eth_hw_addr_set(netdev, mac_addr->sa_data); 2411 2412 return 0; 2413} 2414 2415static int hns3_nic_do_ioctl(struct net_device *netdev, 2416 struct ifreq *ifr, int cmd) 2417{ 2418 struct hnae3_handle *h = hns3_get_handle(netdev); 2419 2420 if (!netif_running(netdev)) 2421 return -EINVAL; 2422 2423 if (!h->ae_algo->ops->do_ioctl) 2424 return -EOPNOTSUPP; 2425 2426 return h->ae_algo->ops->do_ioctl(h, ifr, cmd); 2427} 2428 2429static int hns3_nic_hwtstamp_get(struct net_device *netdev, 2430 struct kernel_hwtstamp_config *config) 2431{ 2432 struct hnae3_handle *h = hns3_get_handle(netdev); 2433 2434 if (!netif_running(netdev)) 2435 return -EINVAL; 2436 2437 if (!h->ae_algo->ops->hwtstamp_get) 2438 return -EOPNOTSUPP; 2439 2440 return h->ae_algo->ops->hwtstamp_get(h, config); 2441} 2442 2443static int hns3_nic_hwtstamp_set(struct net_device *netdev, 2444 struct kernel_hwtstamp_config *config, 2445 struct netlink_ext_ack *extack) 2446{ 2447 struct hnae3_handle *h = hns3_get_handle(netdev); 2448 2449 if (!netif_running(netdev)) 2450 return -EINVAL; 2451 2452 if (!h->ae_algo->ops->hwtstamp_set) 2453 return -EOPNOTSUPP; 2454 2455 return h->ae_algo->ops->hwtstamp_set(h, config, extack); 2456} 2457 2458static int hns3_nic_set_features(struct net_device *netdev, 2459 netdev_features_t features) 2460{ 2461 netdev_features_t changed = netdev->features ^ features; 2462 struct hns3_nic_priv *priv = netdev_priv(netdev); 2463 struct hnae3_handle *h = priv->ae_handle; 2464 bool enable; 2465 int ret; 2466 2467 if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) { 2468 enable = !!(features & NETIF_F_GRO_HW); 2469 ret = h->ae_algo->ops->set_gro_en(h, enable); 2470 if (ret) 2471 return ret; 2472 } 2473 2474 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && 2475 h->ae_algo->ops->enable_hw_strip_rxvtag) { 2476 enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX); 2477 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable); 2478 if (ret) 2479 return ret; 2480 } 2481 2482 if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) { 2483 enable = !!(features & NETIF_F_NTUPLE); 2484 h->ae_algo->ops->enable_fd(h, enable); 2485 } 2486 2487 if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) && 2488 h->ae_algo->ops->cls_flower_active(h)) { 2489 netdev_err(netdev, 2490 "there are offloaded TC filters active, cannot disable HW TC offload\n"); 2491 return -EINVAL; 2492 } 2493 2494 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) && 2495 h->ae_algo->ops->enable_vlan_filter) { 2496 enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER); 2497 ret = h->ae_algo->ops->enable_vlan_filter(h, enable); 2498 if (ret) 2499 return ret; 2500 } 2501 2502 return 0; 2503} 2504 2505static netdev_features_t hns3_features_check(struct sk_buff *skb, 2506 struct net_device *dev, 2507 netdev_features_t features) 2508{ 2509#define HNS3_MAX_HDR_LEN 480U 2510#define HNS3_MAX_L4_HDR_LEN 60U 2511 2512 size_t len; 2513 2514 if (skb->ip_summed != CHECKSUM_PARTIAL) 2515 return features; 2516 2517 if (skb->encapsulation) 2518 len = skb_inner_transport_offset(skb); 2519 else 2520 len = skb_transport_offset(skb); 2521 2522 /* Assume L4 is 60 byte as TCP is the only protocol with a 2523 * a flexible value, and it's max len is 60 bytes. 2524 */ 2525 len += HNS3_MAX_L4_HDR_LEN; 2526 2527 /* Hardware only supports checksum on the skb with a max header 2528 * len of 480 bytes. 2529 */ 2530 if (len > HNS3_MAX_HDR_LEN) 2531 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2532 2533 return features; 2534} 2535 2536static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, 2537 struct hns3_enet_ring *ring, bool is_tx) 2538{ 2539 struct ring_stats ring_stats; 2540 unsigned int start; 2541 2542 do { 2543 start = u64_stats_fetch_begin(&ring->syncp); 2544 ring_stats = ring->stats; 2545 } while (u64_stats_fetch_retry(&ring->syncp, start)); 2546 2547 if (is_tx) { 2548 stats->tx_bytes += ring_stats.tx_bytes; 2549 stats->tx_packets += ring_stats.tx_pkts; 2550 stats->tx_dropped += ring_stats.sw_err_cnt; 2551 stats->tx_dropped += ring_stats.tx_vlan_err; 2552 stats->tx_dropped += ring_stats.tx_l4_proto_err; 2553 stats->tx_dropped += ring_stats.tx_l2l3l4_err; 2554 stats->tx_dropped += ring_stats.tx_tso_err; 2555 stats->tx_dropped += ring_stats.over_max_recursion; 2556 stats->tx_dropped += ring_stats.hw_limitation; 2557 stats->tx_dropped += ring_stats.copy_bits_err; 2558 stats->tx_dropped += ring_stats.skb2sgl_err; 2559 stats->tx_dropped += ring_stats.map_sg_err; 2560 stats->tx_errors += ring_stats.sw_err_cnt; 2561 stats->tx_errors += ring_stats.tx_vlan_err; 2562 stats->tx_errors += ring_stats.tx_l4_proto_err; 2563 stats->tx_errors += ring_stats.tx_l2l3l4_err; 2564 stats->tx_errors += ring_stats.tx_tso_err; 2565 stats->tx_errors += ring_stats.over_max_recursion; 2566 stats->tx_errors += ring_stats.hw_limitation; 2567 stats->tx_errors += ring_stats.copy_bits_err; 2568 stats->tx_errors += ring_stats.skb2sgl_err; 2569 stats->tx_errors += ring_stats.map_sg_err; 2570 } else { 2571 stats->rx_bytes += ring_stats.rx_bytes; 2572 stats->rx_packets += ring_stats.rx_pkts; 2573 stats->rx_dropped += ring_stats.l2_err; 2574 stats->rx_errors += ring_stats.l2_err; 2575 stats->rx_errors += ring_stats.l3l4_csum_err; 2576 stats->rx_crc_errors += ring_stats.l2_err; 2577 stats->multicast += ring_stats.rx_multicast; 2578 stats->rx_length_errors += ring_stats.err_pkt_len; 2579 } 2580} 2581 2582static void hns3_nic_get_stats64(struct net_device *netdev, 2583 struct rtnl_link_stats64 *stats) 2584{ 2585 struct hns3_nic_priv *priv = netdev_priv(netdev); 2586 int queue_num = priv->ae_handle->kinfo.num_tqps; 2587 struct hnae3_handle *handle = priv->ae_handle; 2588 struct rtnl_link_stats64 ring_total_stats; 2589 struct hns3_enet_ring *ring; 2590 int idx; 2591 2592 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 2593 return; 2594 2595 handle->ae_algo->ops->update_stats(handle); 2596 2597 memset(&ring_total_stats, 0, sizeof(ring_total_stats)); 2598 for (idx = 0; idx < queue_num; idx++) { 2599 /* fetch the tx stats */ 2600 ring = &priv->ring[idx]; 2601 hns3_fetch_stats(&ring_total_stats, ring, true); 2602 2603 /* fetch the rx stats */ 2604 ring = &priv->ring[idx + queue_num]; 2605 hns3_fetch_stats(&ring_total_stats, ring, false); 2606 } 2607 2608 stats->tx_bytes = ring_total_stats.tx_bytes; 2609 stats->tx_packets = ring_total_stats.tx_packets; 2610 stats->rx_bytes = ring_total_stats.rx_bytes; 2611 stats->rx_packets = ring_total_stats.rx_packets; 2612 2613 stats->rx_errors = ring_total_stats.rx_errors; 2614 stats->multicast = ring_total_stats.multicast; 2615 stats->rx_length_errors = ring_total_stats.rx_length_errors; 2616 stats->rx_crc_errors = ring_total_stats.rx_crc_errors; 2617 stats->rx_missed_errors = netdev->stats.rx_missed_errors; 2618 2619 stats->tx_errors = ring_total_stats.tx_errors; 2620 stats->rx_dropped = ring_total_stats.rx_dropped; 2621 stats->tx_dropped = ring_total_stats.tx_dropped; 2622 stats->collisions = netdev->stats.collisions; 2623 stats->rx_over_errors = netdev->stats.rx_over_errors; 2624 stats->rx_frame_errors = netdev->stats.rx_frame_errors; 2625 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors; 2626 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors; 2627 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors; 2628 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors; 2629 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors; 2630 stats->tx_window_errors = netdev->stats.tx_window_errors; 2631 stats->rx_compressed = netdev->stats.rx_compressed; 2632 stats->tx_compressed = netdev->stats.tx_compressed; 2633} 2634 2635static int hns3_setup_tc(struct net_device *netdev, void *type_data) 2636{ 2637 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 2638 struct hnae3_knic_private_info *kinfo; 2639 u8 tc = mqprio_qopt->qopt.num_tc; 2640 u16 mode = mqprio_qopt->mode; 2641 u8 hw = mqprio_qopt->qopt.hw; 2642 struct hnae3_handle *h; 2643 2644 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS && 2645 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0))) 2646 return -EOPNOTSUPP; 2647 2648 if (tc > HNAE3_MAX_TC) 2649 return -EINVAL; 2650 2651 if (!netdev) 2652 return -EINVAL; 2653 2654 h = hns3_get_handle(netdev); 2655 kinfo = &h->kinfo; 2656 2657 netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc); 2658 2659 return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ? 2660 kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP; 2661} 2662 2663static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv, 2664 struct flow_cls_offload *flow) 2665{ 2666 int tc = tc_classid_to_hwtc(priv->netdev, flow->classid); 2667 struct hnae3_handle *h = hns3_get_handle(priv->netdev); 2668 2669 switch (flow->command) { 2670 case FLOW_CLS_REPLACE: 2671 if (h->ae_algo->ops->add_cls_flower) 2672 return h->ae_algo->ops->add_cls_flower(h, flow, tc); 2673 break; 2674 case FLOW_CLS_DESTROY: 2675 if (h->ae_algo->ops->del_cls_flower) 2676 return h->ae_algo->ops->del_cls_flower(h, flow); 2677 break; 2678 default: 2679 break; 2680 } 2681 2682 return -EOPNOTSUPP; 2683} 2684 2685static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 2686 void *cb_priv) 2687{ 2688 struct hns3_nic_priv *priv = cb_priv; 2689 2690 if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data)) 2691 return -EOPNOTSUPP; 2692 2693 switch (type) { 2694 case TC_SETUP_CLSFLOWER: 2695 return hns3_setup_tc_cls_flower(priv, type_data); 2696 default: 2697 return -EOPNOTSUPP; 2698 } 2699} 2700 2701static LIST_HEAD(hns3_block_cb_list); 2702 2703static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type, 2704 void *type_data) 2705{ 2706 struct hns3_nic_priv *priv = netdev_priv(dev); 2707 int ret; 2708 2709 switch (type) { 2710 case TC_SETUP_QDISC_MQPRIO: 2711 ret = hns3_setup_tc(dev, type_data); 2712 break; 2713 case TC_SETUP_BLOCK: 2714 ret = flow_block_cb_setup_simple(type_data, 2715 &hns3_block_cb_list, 2716 hns3_setup_tc_block_cb, 2717 priv, priv, true); 2718 break; 2719 default: 2720 return -EOPNOTSUPP; 2721 } 2722 2723 return ret; 2724} 2725 2726static int hns3_vlan_rx_add_vid(struct net_device *netdev, 2727 __be16 proto, u16 vid) 2728{ 2729 struct hnae3_handle *h = hns3_get_handle(netdev); 2730 int ret = -EIO; 2731 2732 if (h->ae_algo->ops->set_vlan_filter) 2733 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false); 2734 2735 return ret; 2736} 2737 2738static int hns3_vlan_rx_kill_vid(struct net_device *netdev, 2739 __be16 proto, u16 vid) 2740{ 2741 struct hnae3_handle *h = hns3_get_handle(netdev); 2742 int ret = -EIO; 2743 2744 if (h->ae_algo->ops->set_vlan_filter) 2745 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true); 2746 2747 return ret; 2748} 2749 2750static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, 2751 u8 qos, __be16 vlan_proto) 2752{ 2753 struct hnae3_handle *h = hns3_get_handle(netdev); 2754 int ret = -EIO; 2755 2756 netif_dbg(h, drv, netdev, 2757 "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n", 2758 vf, vlan, qos, ntohs(vlan_proto)); 2759 2760 if (h->ae_algo->ops->set_vf_vlan_filter) 2761 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan, 2762 qos, vlan_proto); 2763 2764 return ret; 2765} 2766 2767static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) 2768{ 2769 struct hnae3_handle *handle = hns3_get_handle(netdev); 2770 2771 if (hns3_nic_resetting(netdev)) 2772 return -EBUSY; 2773 2774 if (!handle->ae_algo->ops->set_vf_spoofchk) 2775 return -EOPNOTSUPP; 2776 2777 return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable); 2778} 2779 2780static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable) 2781{ 2782 struct hnae3_handle *handle = hns3_get_handle(netdev); 2783 2784 if (!handle->ae_algo->ops->set_vf_trust) 2785 return -EOPNOTSUPP; 2786 2787 return handle->ae_algo->ops->set_vf_trust(handle, vf, enable); 2788} 2789 2790static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu) 2791{ 2792 struct hnae3_handle *h = hns3_get_handle(netdev); 2793 int ret; 2794 2795 if (hns3_nic_resetting(netdev)) 2796 return -EBUSY; 2797 2798 if (!h->ae_algo->ops->set_mtu) 2799 return -EOPNOTSUPP; 2800 2801 netif_dbg(h, drv, netdev, 2802 "change mtu from %u to %d\n", netdev->mtu, new_mtu); 2803 2804 ret = h->ae_algo->ops->set_mtu(h, new_mtu); 2805 if (ret) 2806 netdev_err(netdev, "failed to change MTU in hardware %d\n", 2807 ret); 2808 else 2809 WRITE_ONCE(netdev->mtu, new_mtu); 2810 2811 return ret; 2812} 2813 2814static int hns3_get_timeout_queue(struct net_device *ndev) 2815{ 2816 unsigned int i; 2817 2818 /* Find the stopped queue the same way the stack does */ 2819 for (i = 0; i < ndev->num_tx_queues; i++) { 2820 struct netdev_queue *q; 2821 unsigned long trans_start; 2822 2823 q = netdev_get_tx_queue(ndev, i); 2824 trans_start = READ_ONCE(q->trans_start); 2825 if (netif_xmit_stopped(q) && 2826 time_after(jiffies, 2827 (trans_start + ndev->watchdog_timeo))) { 2828#ifdef CONFIG_BQL 2829 struct dql *dql = &q->dql; 2830 2831 netdev_info(ndev, "DQL info last_cnt: %u, queued: %u, adj_limit: %u, completed: %u\n", 2832 dql->last_obj_cnt, dql->num_queued, 2833 dql->adj_limit, dql->num_completed); 2834#endif 2835 netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n", 2836 q->state, 2837 jiffies_to_msecs(jiffies - trans_start)); 2838 break; 2839 } 2840 } 2841 2842 return i; 2843} 2844 2845static void hns3_dump_queue_stats(struct net_device *ndev, 2846 struct hns3_enet_ring *tx_ring, 2847 int timeout_queue) 2848{ 2849 struct napi_struct *napi = &tx_ring->tqp_vector->napi; 2850 struct hns3_nic_priv *priv = netdev_priv(ndev); 2851 2852 netdev_info(ndev, 2853 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n", 2854 priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use, 2855 tx_ring->next_to_clean, napi->state); 2856 2857 netdev_info(ndev, 2858 "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n", 2859 tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes, 2860 tx_ring->stats.sw_err_cnt, tx_ring->pending_buf); 2861 2862 netdev_info(ndev, 2863 "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n", 2864 tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more, 2865 tx_ring->stats.restart_queue, tx_ring->stats.tx_busy); 2866 2867 netdev_info(ndev, "tx_push: %llu, tx_mem_doorbell: %llu\n", 2868 tx_ring->stats.tx_push, tx_ring->stats.tx_mem_doorbell); 2869} 2870 2871static void hns3_dump_queue_reg(struct net_device *ndev, 2872 struct hns3_enet_ring *tx_ring) 2873{ 2874 netdev_info(ndev, 2875 "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n", 2876 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_NUM_REG), 2877 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_HEAD_REG), 2878 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TAIL_REG), 2879 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_ERR_REG), 2880 readl(tx_ring->tqp_vector->mask_addr)); 2881 netdev_info(ndev, 2882 "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n", 2883 hns3_tqp_read_reg(tx_ring, HNS3_RING_EN_REG), 2884 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TC_REG), 2885 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_FBDNUM_REG), 2886 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_OFFSET_REG), 2887 hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_EBDNUM_REG), 2888 hns3_tqp_read_reg(tx_ring, 2889 HNS3_RING_TX_RING_EBD_OFFSET_REG)); 2890} 2891 2892static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) 2893{ 2894 struct hns3_nic_priv *priv = netdev_priv(ndev); 2895 struct hnae3_handle *h = hns3_get_handle(ndev); 2896 struct hns3_enet_ring *tx_ring; 2897 u32 timeout_queue; 2898 2899 timeout_queue = hns3_get_timeout_queue(ndev); 2900 if (timeout_queue >= ndev->num_tx_queues) { 2901 netdev_info(ndev, 2902 "no netdev TX timeout queue found, timeout count: %llu\n", 2903 priv->tx_timeout_count); 2904 return false; 2905 } 2906 2907 priv->tx_timeout_count++; 2908 2909 tx_ring = &priv->ring[timeout_queue]; 2910 hns3_dump_queue_stats(ndev, tx_ring, timeout_queue); 2911 2912 /* When mac received many pause frames continuous, it's unable to send 2913 * packets, which may cause tx timeout 2914 */ 2915 if (h->ae_algo->ops->get_mac_stats) { 2916 struct hns3_mac_stats mac_stats; 2917 2918 h->ae_algo->ops->get_mac_stats(h, &mac_stats); 2919 netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n", 2920 mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt); 2921 } 2922 2923 hns3_dump_queue_reg(ndev, tx_ring); 2924 2925 return true; 2926} 2927 2928static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue) 2929{ 2930 struct hns3_nic_priv *priv = netdev_priv(ndev); 2931 struct hnae3_handle *h = priv->ae_handle; 2932 2933 if (!hns3_get_tx_timeo_queue_info(ndev)) 2934 return; 2935 2936 /* request the reset, and let the hclge to determine 2937 * which reset level should be done 2938 */ 2939 if (h->ae_algo->ops->reset_event) 2940 h->ae_algo->ops->reset_event(h->pdev, h); 2941} 2942 2943#ifdef CONFIG_RFS_ACCEL 2944static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 2945 u16 rxq_index, u32 flow_id) 2946{ 2947 struct hnae3_handle *h = hns3_get_handle(dev); 2948 struct flow_keys fkeys; 2949 2950 if (!h->ae_algo->ops->add_arfs_entry) 2951 return -EOPNOTSUPP; 2952 2953 if (skb->encapsulation) 2954 return -EPROTONOSUPPORT; 2955 2956 if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0)) 2957 return -EPROTONOSUPPORT; 2958 2959 if ((fkeys.basic.n_proto != htons(ETH_P_IP) && 2960 fkeys.basic.n_proto != htons(ETH_P_IPV6)) || 2961 (fkeys.basic.ip_proto != IPPROTO_TCP && 2962 fkeys.basic.ip_proto != IPPROTO_UDP)) 2963 return -EPROTONOSUPPORT; 2964 2965 return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys); 2966} 2967#endif 2968 2969static int hns3_nic_get_vf_config(struct net_device *ndev, int vf, 2970 struct ifla_vf_info *ivf) 2971{ 2972 struct hnae3_handle *h = hns3_get_handle(ndev); 2973 2974 if (!h->ae_algo->ops->get_vf_config) 2975 return -EOPNOTSUPP; 2976 2977 return h->ae_algo->ops->get_vf_config(h, vf, ivf); 2978} 2979 2980static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf, 2981 int link_state) 2982{ 2983 struct hnae3_handle *h = hns3_get_handle(ndev); 2984 2985 if (!h->ae_algo->ops->set_vf_link_state) 2986 return -EOPNOTSUPP; 2987 2988 return h->ae_algo->ops->set_vf_link_state(h, vf, link_state); 2989} 2990 2991static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf, 2992 int min_tx_rate, int max_tx_rate) 2993{ 2994 struct hnae3_handle *h = hns3_get_handle(ndev); 2995 2996 if (!h->ae_algo->ops->set_vf_rate) 2997 return -EOPNOTSUPP; 2998 2999 return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate, 3000 false); 3001} 3002 3003static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) 3004{ 3005 struct hnae3_handle *h = hns3_get_handle(netdev); 3006 char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; 3007 3008 if (!h->ae_algo->ops->set_vf_mac) 3009 return -EOPNOTSUPP; 3010 3011 if (is_multicast_ether_addr(mac)) { 3012 hnae3_format_mac_addr(format_mac_addr, mac); 3013 netdev_err(netdev, 3014 "Invalid MAC:%s specified. Could not set MAC\n", 3015 format_mac_addr); 3016 return -EINVAL; 3017 } 3018 3019 return h->ae_algo->ops->set_vf_mac(h, vf_id, mac); 3020} 3021 3022#define HNS3_INVALID_DSCP 0xff 3023#define HNS3_DSCP_SHIFT 2 3024 3025static u8 hns3_get_skb_dscp(struct sk_buff *skb) 3026{ 3027 __be16 protocol = skb->protocol; 3028 u8 dscp = HNS3_INVALID_DSCP; 3029 3030 if (protocol == htons(ETH_P_8021Q)) 3031 protocol = vlan_get_protocol(skb); 3032 3033 if (protocol == htons(ETH_P_IP)) 3034 dscp = ipv4_get_dsfield(ip_hdr(skb)) >> HNS3_DSCP_SHIFT; 3035 else if (protocol == htons(ETH_P_IPV6)) 3036 dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> HNS3_DSCP_SHIFT; 3037 3038 return dscp; 3039} 3040 3041static u16 hns3_nic_select_queue(struct net_device *netdev, 3042 struct sk_buff *skb, 3043 struct net_device *sb_dev) 3044{ 3045 struct hnae3_handle *h = hns3_get_handle(netdev); 3046 u8 dscp; 3047 3048 if (h->kinfo.tc_map_mode != HNAE3_TC_MAP_MODE_DSCP || 3049 !h->ae_algo->ops->get_dscp_prio) 3050 goto out; 3051 3052 dscp = hns3_get_skb_dscp(skb); 3053 if (unlikely(dscp >= HNAE3_MAX_DSCP)) 3054 goto out; 3055 3056 skb->priority = h->kinfo.dscp_prio[dscp]; 3057 if (skb->priority == HNAE3_PRIO_ID_INVALID) 3058 skb->priority = 0; 3059 3060out: 3061 return netdev_pick_tx(netdev, skb, sb_dev); 3062} 3063 3064static const struct net_device_ops hns3_nic_netdev_ops = { 3065 .ndo_open = hns3_nic_net_open, 3066 .ndo_stop = hns3_nic_net_stop, 3067 .ndo_start_xmit = hns3_nic_net_xmit, 3068 .ndo_tx_timeout = hns3_nic_net_timeout, 3069 .ndo_set_mac_address = hns3_nic_net_set_mac_address, 3070 .ndo_eth_ioctl = hns3_nic_do_ioctl, 3071 .ndo_change_mtu = hns3_nic_change_mtu, 3072 .ndo_set_features = hns3_nic_set_features, 3073 .ndo_features_check = hns3_features_check, 3074 .ndo_get_stats64 = hns3_nic_get_stats64, 3075 .ndo_setup_tc = hns3_nic_setup_tc, 3076 .ndo_set_rx_mode = hns3_nic_set_rx_mode, 3077 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid, 3078 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid, 3079 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan, 3080 .ndo_set_vf_spoofchk = hns3_set_vf_spoofchk, 3081 .ndo_set_vf_trust = hns3_set_vf_trust, 3082#ifdef CONFIG_RFS_ACCEL 3083 .ndo_rx_flow_steer = hns3_rx_flow_steer, 3084#endif 3085 .ndo_get_vf_config = hns3_nic_get_vf_config, 3086 .ndo_set_vf_link_state = hns3_nic_set_vf_link_state, 3087 .ndo_set_vf_rate = hns3_nic_set_vf_rate, 3088 .ndo_set_vf_mac = hns3_nic_set_vf_mac, 3089 .ndo_select_queue = hns3_nic_select_queue, 3090 .ndo_hwtstamp_get = hns3_nic_hwtstamp_get, 3091 .ndo_hwtstamp_set = hns3_nic_hwtstamp_set, 3092}; 3093 3094bool hns3_is_phys_func(struct pci_dev *pdev) 3095{ 3096 u32 dev_id = pdev->device; 3097 3098 switch (dev_id) { 3099 case HNAE3_DEV_ID_GE: 3100 case HNAE3_DEV_ID_25GE: 3101 case HNAE3_DEV_ID_25GE_RDMA: 3102 case HNAE3_DEV_ID_25GE_RDMA_MACSEC: 3103 case HNAE3_DEV_ID_50GE_RDMA: 3104 case HNAE3_DEV_ID_50GE_RDMA_MACSEC: 3105 case HNAE3_DEV_ID_100G_RDMA_MACSEC: 3106 case HNAE3_DEV_ID_200G_RDMA: 3107 return true; 3108 case HNAE3_DEV_ID_VF: 3109 case HNAE3_DEV_ID_RDMA_DCB_PFC_VF: 3110 return false; 3111 default: 3112 dev_warn(&pdev->dev, "un-recognized pci device-id %u", 3113 dev_id); 3114 } 3115 3116 return false; 3117} 3118 3119static void hns3_disable_sriov(struct pci_dev *pdev) 3120{ 3121 /* If our VFs are assigned we cannot shut down SR-IOV 3122 * without causing issues, so just leave the hardware 3123 * available but disabled 3124 */ 3125 if (pci_vfs_assigned(pdev)) { 3126 dev_warn(&pdev->dev, 3127 "disabling driver while VFs are assigned\n"); 3128 return; 3129 } 3130 3131 pci_disable_sriov(pdev); 3132} 3133 3134/* hns3_probe - Device initialization routine 3135 * @pdev: PCI device information struct 3136 * @ent: entry in hns3_pci_tbl 3137 * 3138 * hns3_probe initializes a PF identified by a pci_dev structure. 3139 * The OS initialization, configuring of the PF private structure, 3140 * and a hardware reset occur. 3141 * 3142 * Returns 0 on success, negative on failure 3143 */ 3144static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3145{ 3146 struct hnae3_ae_dev *ae_dev; 3147 int ret; 3148 3149 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL); 3150 if (!ae_dev) 3151 return -ENOMEM; 3152 3153 ae_dev->pdev = pdev; 3154 ae_dev->flag = ent->driver_data; 3155 pci_set_drvdata(pdev, ae_dev); 3156 3157 ret = hnae3_register_ae_dev(ae_dev); 3158 if (ret) 3159 pci_set_drvdata(pdev, NULL); 3160 3161 return ret; 3162} 3163 3164/** 3165 * hns3_clean_vf_config 3166 * @pdev: pointer to a pci_dev structure 3167 * @num_vfs: number of VFs allocated 3168 * 3169 * Clean residual vf config after disable sriov 3170 **/ 3171static void hns3_clean_vf_config(struct pci_dev *pdev, int num_vfs) 3172{ 3173 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3174 3175 if (ae_dev->ops->clean_vf_config) 3176 ae_dev->ops->clean_vf_config(ae_dev, num_vfs); 3177} 3178 3179/* hns3_remove - Device removal routine 3180 * @pdev: PCI device information struct 3181 */ 3182static void hns3_remove(struct pci_dev *pdev) 3183{ 3184 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3185 3186 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV)) 3187 hns3_disable_sriov(pdev); 3188 3189 hnae3_unregister_ae_dev(ae_dev); 3190 pci_set_drvdata(pdev, NULL); 3191} 3192 3193/** 3194 * hns3_pci_sriov_configure 3195 * @pdev: pointer to a pci_dev structure 3196 * @num_vfs: number of VFs to allocate 3197 * 3198 * Enable or change the number of VFs. Called when the user updates the number 3199 * of VFs in sysfs. 3200 **/ 3201static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 3202{ 3203 int ret; 3204 3205 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) { 3206 dev_warn(&pdev->dev, "Can not config SRIOV\n"); 3207 return -EINVAL; 3208 } 3209 3210 if (num_vfs) { 3211 ret = pci_enable_sriov(pdev, num_vfs); 3212 if (ret) 3213 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret); 3214 else 3215 return num_vfs; 3216 } else if (!pci_vfs_assigned(pdev)) { 3217 int num_vfs_pre = pci_num_vf(pdev); 3218 3219 pci_disable_sriov(pdev); 3220 hns3_clean_vf_config(pdev, num_vfs_pre); 3221 } else { 3222 dev_warn(&pdev->dev, 3223 "Unable to free VFs because some are assigned to VMs.\n"); 3224 } 3225 3226 return 0; 3227} 3228 3229static void hns3_shutdown(struct pci_dev *pdev) 3230{ 3231 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3232 3233 hnae3_unregister_ae_dev(ae_dev); 3234 pci_set_drvdata(pdev, NULL); 3235 3236 if (system_state == SYSTEM_POWER_OFF) 3237 pci_set_power_state(pdev, PCI_D3hot); 3238} 3239 3240static int __maybe_unused hns3_suspend(struct device *dev) 3241{ 3242 struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev); 3243 3244 if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) { 3245 dev_info(dev, "Begin to suspend.\n"); 3246 if (ae_dev->ops && ae_dev->ops->reset_prepare) 3247 ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET); 3248 } 3249 3250 return 0; 3251} 3252 3253static int __maybe_unused hns3_resume(struct device *dev) 3254{ 3255 struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev); 3256 3257 if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) { 3258 dev_info(dev, "Begin to resume.\n"); 3259 if (ae_dev->ops && ae_dev->ops->reset_done) 3260 ae_dev->ops->reset_done(ae_dev); 3261 } 3262 3263 return 0; 3264} 3265 3266static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev, 3267 pci_channel_state_t state) 3268{ 3269 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3270 pci_ers_result_t ret; 3271 3272 dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state); 3273 3274 if (state == pci_channel_io_perm_failure) 3275 return PCI_ERS_RESULT_DISCONNECT; 3276 3277 if (!ae_dev || !ae_dev->ops) { 3278 dev_err(&pdev->dev, 3279 "Can't recover - error happened before device initialized\n"); 3280 return PCI_ERS_RESULT_NONE; 3281 } 3282 3283 if (ae_dev->ops->handle_hw_ras_error) 3284 ret = ae_dev->ops->handle_hw_ras_error(ae_dev); 3285 else 3286 return PCI_ERS_RESULT_NONE; 3287 3288 return ret; 3289} 3290 3291static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev) 3292{ 3293 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3294 const struct hnae3_ae_ops *ops; 3295 enum hnae3_reset_type reset_type; 3296 struct device *dev = &pdev->dev; 3297 3298 if (!ae_dev || !ae_dev->ops) 3299 return PCI_ERS_RESULT_NONE; 3300 3301 ops = ae_dev->ops; 3302 /* request the reset */ 3303 if (ops->reset_event && ops->get_reset_level && 3304 ops->set_default_reset_request) { 3305 if (ae_dev->hw_err_reset_req) { 3306 reset_type = ops->get_reset_level(ae_dev, 3307 &ae_dev->hw_err_reset_req); 3308 ops->set_default_reset_request(ae_dev, reset_type); 3309 dev_info(dev, "requesting reset due to PCI error\n"); 3310 ops->reset_event(pdev, NULL); 3311 } 3312 3313 return PCI_ERS_RESULT_RECOVERED; 3314 } 3315 3316 return PCI_ERS_RESULT_DISCONNECT; 3317} 3318 3319static void hns3_reset_prepare(struct pci_dev *pdev) 3320{ 3321 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3322 3323 dev_info(&pdev->dev, "FLR prepare\n"); 3324 if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare) 3325 ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET); 3326} 3327 3328static void hns3_reset_done(struct pci_dev *pdev) 3329{ 3330 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3331 3332 dev_info(&pdev->dev, "FLR done\n"); 3333 if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done) 3334 ae_dev->ops->reset_done(ae_dev); 3335} 3336 3337static const struct pci_error_handlers hns3_err_handler = { 3338 .error_detected = hns3_error_detected, 3339 .slot_reset = hns3_slot_reset, 3340 .reset_prepare = hns3_reset_prepare, 3341 .reset_done = hns3_reset_done, 3342}; 3343 3344static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume); 3345 3346static struct pci_driver hns3_driver = { 3347 .name = hns3_driver_name, 3348 .id_table = hns3_pci_tbl, 3349 .probe = hns3_probe, 3350 .remove = hns3_remove, 3351 .shutdown = hns3_shutdown, 3352 .driver.pm = &hns3_pm_ops, 3353 .sriov_configure = hns3_pci_sriov_configure, 3354 .err_handler = &hns3_err_handler, 3355}; 3356 3357/* set default feature to hns3 */ 3358static void hns3_set_default_feature(struct net_device *netdev) 3359{ 3360 struct hnae3_handle *h = hns3_get_handle(netdev); 3361 struct pci_dev *pdev = h->pdev; 3362 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 3363 3364 netdev->priv_flags |= IFF_UNICAST_FLT; 3365 3366 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 3367 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 3368 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO | 3369 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE | 3370 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | 3371 NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST; 3372 3373 if (hnae3_ae_dev_gro_supported(ae_dev)) 3374 netdev->features |= NETIF_F_GRO_HW; 3375 3376 if (hnae3_ae_dev_fd_supported(ae_dev)) 3377 netdev->features |= NETIF_F_NTUPLE; 3378 3379 if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) 3380 netdev->features |= NETIF_F_GSO_UDP_L4; 3381 3382 if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) 3383 netdev->features |= NETIF_F_HW_CSUM; 3384 else 3385 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3386 3387 if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps)) 3388 netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 3389 3390 if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) 3391 netdev->features |= NETIF_F_HW_TC; 3392 3393 netdev->hw_features |= netdev->features; 3394 if (!test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps)) 3395 netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 3396 3397 netdev->vlan_features |= netdev->features & 3398 ~(NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX | 3399 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_GRO_HW | NETIF_F_NTUPLE | 3400 NETIF_F_HW_TC); 3401 3402 netdev->hw_enc_features |= netdev->vlan_features | NETIF_F_TSO_MANGLEID; 3403 3404 /* The device_version V3 hardware can't offload the checksum for IP in 3405 * GRE packets, but can do it for NvGRE. So default to disable the 3406 * checksum and GSO offload for GRE. 3407 */ 3408 if (ae_dev->dev_version > HNAE3_DEVICE_VERSION_V2) { 3409 netdev->features &= ~NETIF_F_GSO_GRE; 3410 netdev->features &= ~NETIF_F_GSO_GRE_CSUM; 3411 } 3412} 3413 3414static int hns3_alloc_buffer(struct hns3_enet_ring *ring, 3415 struct hns3_desc_cb *cb) 3416{ 3417 unsigned int order = hns3_page_order(ring); 3418 struct page *p; 3419 3420 if (ring->page_pool) { 3421 p = page_pool_dev_alloc_frag(ring->page_pool, 3422 &cb->page_offset, 3423 hns3_buf_size(ring)); 3424 if (unlikely(!p)) 3425 return -ENOMEM; 3426 3427 cb->priv = p; 3428 cb->buf = page_address(p); 3429 cb->dma = page_pool_get_dma_addr(p); 3430 cb->type = DESC_TYPE_PP_FRAG; 3431 cb->reuse_flag = 0; 3432 return 0; 3433 } 3434 3435 p = dev_alloc_pages(order); 3436 if (!p) 3437 return -ENOMEM; 3438 3439 cb->priv = p; 3440 cb->page_offset = 0; 3441 cb->reuse_flag = 0; 3442 cb->buf = page_address(p); 3443 cb->length = hns3_page_size(ring); 3444 cb->type = DESC_TYPE_PAGE; 3445 page_ref_add(p, USHRT_MAX - 1); 3446 cb->pagecnt_bias = USHRT_MAX; 3447 3448 return 0; 3449} 3450 3451static void hns3_free_buffer(struct hns3_enet_ring *ring, 3452 struct hns3_desc_cb *cb, int budget) 3453{ 3454 if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_HEAD | 3455 DESC_TYPE_BOUNCE_ALL | DESC_TYPE_SGL_SKB)) 3456 napi_consume_skb(cb->priv, budget); 3457 else if (!HNAE3_IS_TX_RING(ring)) { 3458 if (cb->type & DESC_TYPE_PAGE && cb->pagecnt_bias) 3459 __page_frag_cache_drain(cb->priv, cb->pagecnt_bias); 3460 else if (cb->type & DESC_TYPE_PP_FRAG) 3461 page_pool_put_full_page(ring->page_pool, cb->priv, 3462 false); 3463 } 3464 memset(cb, 0, sizeof(*cb)); 3465} 3466 3467static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb) 3468{ 3469 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0, 3470 cb->length, ring_to_dma_dir(ring)); 3471 3472 if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma))) 3473 return -EIO; 3474 3475 return 0; 3476} 3477 3478static void hns3_unmap_buffer(struct hns3_enet_ring *ring, 3479 struct hns3_desc_cb *cb) 3480{ 3481 if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB)) 3482 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length, 3483 ring_to_dma_dir(ring)); 3484 else if ((cb->type & DESC_TYPE_PAGE) && cb->length) 3485 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length, 3486 ring_to_dma_dir(ring)); 3487 else if (cb->type & (DESC_TYPE_BOUNCE_ALL | DESC_TYPE_BOUNCE_HEAD | 3488 DESC_TYPE_SGL_SKB)) 3489 hns3_tx_spare_reclaim_cb(ring, cb); 3490} 3491 3492static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i) 3493{ 3494 hns3_unmap_buffer(ring, &ring->desc_cb[i]); 3495 ring->desc[i].addr = 0; 3496 ring->desc_cb[i].refill = 0; 3497} 3498 3499static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i, 3500 int budget) 3501{ 3502 struct hns3_desc_cb *cb = &ring->desc_cb[i]; 3503 3504 if (!ring->desc_cb[i].dma) 3505 return; 3506 3507 hns3_buffer_detach(ring, i); 3508 hns3_free_buffer(ring, cb, budget); 3509} 3510 3511static void hns3_free_buffers(struct hns3_enet_ring *ring) 3512{ 3513 int i; 3514 3515 for (i = 0; i < ring->desc_num; i++) 3516 hns3_free_buffer_detach(ring, i, 0); 3517} 3518 3519/* free desc along with its attached buffer */ 3520static void hns3_free_desc(struct hns3_enet_ring *ring) 3521{ 3522 int size = ring->desc_num * sizeof(ring->desc[0]); 3523 3524 hns3_free_buffers(ring); 3525 3526 if (ring->desc) { 3527 dma_free_coherent(ring_to_dev(ring), size, 3528 ring->desc, ring->desc_dma_addr); 3529 ring->desc = NULL; 3530 } 3531} 3532 3533static int hns3_alloc_desc(struct hns3_enet_ring *ring) 3534{ 3535 int size = ring->desc_num * sizeof(ring->desc[0]); 3536 3537 ring->desc = dma_alloc_coherent(ring_to_dev(ring), size, 3538 &ring->desc_dma_addr, GFP_KERNEL); 3539 if (!ring->desc) 3540 return -ENOMEM; 3541 3542 return 0; 3543} 3544 3545static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring, 3546 struct hns3_desc_cb *cb) 3547{ 3548 int ret; 3549 3550 ret = hns3_alloc_buffer(ring, cb); 3551 if (ret || ring->page_pool) 3552 goto out; 3553 3554 ret = hns3_map_buffer(ring, cb); 3555 if (ret) 3556 goto out_with_buf; 3557 3558 return 0; 3559 3560out_with_buf: 3561 hns3_free_buffer(ring, cb, 0); 3562out: 3563 return ret; 3564} 3565 3566static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i) 3567{ 3568 int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]); 3569 3570 if (ret) 3571 return ret; 3572 3573 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma + 3574 ring->desc_cb[i].page_offset); 3575 ring->desc_cb[i].refill = 1; 3576 3577 return 0; 3578} 3579 3580/* Allocate memory for raw pkg, and map with dma */ 3581static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring) 3582{ 3583 int i, j, ret; 3584 3585 for (i = 0; i < ring->desc_num; i++) { 3586 ret = hns3_alloc_and_attach_buffer(ring, i); 3587 if (ret) 3588 goto out_buffer_fail; 3589 3590 if (!(i % HNS3_RESCHED_BD_NUM)) 3591 cond_resched(); 3592 } 3593 3594 return 0; 3595 3596out_buffer_fail: 3597 for (j = i - 1; j >= 0; j--) 3598 hns3_free_buffer_detach(ring, j, 0); 3599 return ret; 3600} 3601 3602/* detach a in-used buffer and replace with a reserved one */ 3603static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i, 3604 struct hns3_desc_cb *res_cb) 3605{ 3606 hns3_unmap_buffer(ring, &ring->desc_cb[i]); 3607 ring->desc_cb[i] = *res_cb; 3608 ring->desc_cb[i].refill = 1; 3609 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma + 3610 ring->desc_cb[i].page_offset); 3611 ring->desc[i].rx.bd_base_info = 0; 3612} 3613 3614static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i) 3615{ 3616 ring->desc_cb[i].reuse_flag = 0; 3617 ring->desc_cb[i].refill = 1; 3618 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma + 3619 ring->desc_cb[i].page_offset); 3620 ring->desc[i].rx.bd_base_info = 0; 3621 3622 dma_sync_single_for_device(ring_to_dev(ring), 3623 ring->desc_cb[i].dma + ring->desc_cb[i].page_offset, 3624 hns3_buf_size(ring), 3625 DMA_FROM_DEVICE); 3626} 3627 3628static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring, 3629 int *bytes, int *pkts, int budget) 3630{ 3631 /* This smp_load_acquire() pairs with smp_store_release() in 3632 * hns3_tx_doorbell(). 3633 */ 3634 int ltu = smp_load_acquire(&ring->last_to_use); 3635 int ntc = ring->next_to_clean; 3636 struct hns3_desc_cb *desc_cb; 3637 bool reclaimed = false; 3638 struct hns3_desc *desc; 3639 3640 while (ltu != ntc) { 3641 desc = &ring->desc[ntc]; 3642 3643 if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) & 3644 BIT(HNS3_TXD_VLD_B)) 3645 break; 3646 3647 desc_cb = &ring->desc_cb[ntc]; 3648 3649 if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_ALL | 3650 DESC_TYPE_BOUNCE_HEAD | 3651 DESC_TYPE_SGL_SKB)) { 3652 (*pkts)++; 3653 (*bytes) += desc_cb->send_bytes; 3654 } 3655 3656 /* desc_cb will be cleaned, after hnae3_free_buffer_detach */ 3657 hns3_free_buffer_detach(ring, ntc, budget); 3658 3659 if (++ntc == ring->desc_num) 3660 ntc = 0; 3661 3662 /* Issue prefetch for next Tx descriptor */ 3663 prefetch(&ring->desc_cb[ntc]); 3664 reclaimed = true; 3665 } 3666 3667 if (unlikely(!reclaimed)) 3668 return false; 3669 3670 /* This smp_store_release() pairs with smp_load_acquire() in 3671 * ring_space called by hns3_nic_net_xmit. 3672 */ 3673 smp_store_release(&ring->next_to_clean, ntc); 3674 3675 hns3_tx_spare_update(ring); 3676 3677 return true; 3678} 3679 3680void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget) 3681{ 3682 struct net_device *netdev = ring_to_netdev(ring); 3683 struct hns3_nic_priv *priv = netdev_priv(netdev); 3684 struct netdev_queue *dev_queue; 3685 int bytes, pkts; 3686 3687 bytes = 0; 3688 pkts = 0; 3689 3690 if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget))) 3691 return; 3692 3693 ring->tqp_vector->tx_group.total_bytes += bytes; 3694 ring->tqp_vector->tx_group.total_packets += pkts; 3695 3696 u64_stats_update_begin(&ring->syncp); 3697 ring->stats.tx_bytes += bytes; 3698 ring->stats.tx_pkts += pkts; 3699 u64_stats_update_end(&ring->syncp); 3700 3701 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index); 3702 netdev_tx_completed_queue(dev_queue, pkts, bytes); 3703 3704 if (unlikely(netif_carrier_ok(netdev) && 3705 ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) { 3706 /* Make sure that anybody stopping the queue after this 3707 * sees the new next_to_clean. 3708 */ 3709 smp_mb(); 3710 if (netif_tx_queue_stopped(dev_queue) && 3711 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) { 3712 netif_tx_wake_queue(dev_queue); 3713 ring->stats.restart_queue++; 3714 } 3715 } 3716} 3717 3718static int hns3_desc_unused(struct hns3_enet_ring *ring) 3719{ 3720 int ntc = ring->next_to_clean; 3721 int ntu = ring->next_to_use; 3722 3723 if (unlikely(ntc == ntu && !ring->desc_cb[ntc].refill)) 3724 return ring->desc_num; 3725 3726 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu; 3727} 3728 3729/* Return true if there is any allocation failure */ 3730static bool hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, 3731 int cleand_count) 3732{ 3733 struct hns3_desc_cb *desc_cb; 3734 struct hns3_desc_cb res_cbs; 3735 int i, ret; 3736 3737 for (i = 0; i < cleand_count; i++) { 3738 desc_cb = &ring->desc_cb[ring->next_to_use]; 3739 if (desc_cb->reuse_flag) { 3740 hns3_ring_stats_update(ring, reuse_pg_cnt); 3741 3742 hns3_reuse_buffer(ring, ring->next_to_use); 3743 } else { 3744 ret = hns3_alloc_and_map_buffer(ring, &res_cbs); 3745 if (ret) { 3746 hns3_ring_stats_update(ring, sw_err_cnt); 3747 3748 hns3_rl_err(ring_to_netdev(ring), 3749 "alloc rx buffer failed: %d\n", 3750 ret); 3751 3752 writel(i, ring->tqp->io_base + 3753 HNS3_RING_RX_RING_HEAD_REG); 3754 return true; 3755 } 3756 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs); 3757 3758 hns3_ring_stats_update(ring, non_reuse_pg); 3759 } 3760 3761 ring_ptr_move_fw(ring, next_to_use); 3762 } 3763 3764 writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG); 3765 return false; 3766} 3767 3768static bool hns3_can_reuse_page(struct hns3_desc_cb *cb) 3769{ 3770 return page_count(cb->priv) == cb->pagecnt_bias; 3771} 3772 3773static int hns3_handle_rx_copybreak(struct sk_buff *skb, int i, 3774 struct hns3_enet_ring *ring, 3775 int pull_len, 3776 struct hns3_desc_cb *desc_cb) 3777{ 3778 struct hns3_desc *desc = &ring->desc[ring->next_to_clean]; 3779 u32 frag_offset = desc_cb->page_offset + pull_len; 3780 int size = le16_to_cpu(desc->rx.size); 3781 u32 frag_size = size - pull_len; 3782 void *frag = napi_alloc_frag(frag_size); 3783 3784 if (unlikely(!frag)) { 3785 hns3_ring_stats_update(ring, frag_alloc_err); 3786 3787 hns3_rl_err(ring_to_netdev(ring), 3788 "failed to allocate rx frag\n"); 3789 return -ENOMEM; 3790 } 3791 3792 desc_cb->reuse_flag = 1; 3793 memcpy(frag, desc_cb->buf + frag_offset, frag_size); 3794 skb_add_rx_frag(skb, i, virt_to_page(frag), 3795 offset_in_page(frag), frag_size, frag_size); 3796 3797 hns3_ring_stats_update(ring, frag_alloc); 3798 return 0; 3799} 3800 3801static void hns3_nic_reuse_page(struct sk_buff *skb, int i, 3802 struct hns3_enet_ring *ring, int pull_len, 3803 struct hns3_desc_cb *desc_cb) 3804{ 3805 struct hns3_desc *desc = &ring->desc[ring->next_to_clean]; 3806 u32 frag_offset = desc_cb->page_offset + pull_len; 3807 int size = le16_to_cpu(desc->rx.size); 3808 u32 truesize = hns3_buf_size(ring); 3809 u32 frag_size = size - pull_len; 3810 int ret = 0; 3811 bool reused; 3812 3813 if (ring->page_pool) { 3814 skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset, 3815 frag_size, truesize); 3816 return; 3817 } 3818 3819 /* Avoid re-using remote or pfmem page */ 3820 if (unlikely(!dev_page_is_reusable(desc_cb->priv))) 3821 goto out; 3822 3823 reused = hns3_can_reuse_page(desc_cb); 3824 3825 /* Rx page can be reused when: 3826 * 1. Rx page is only owned by the driver when page_offset 3827 * is zero, which means 0 @ truesize will be used by 3828 * stack after skb_add_rx_frag() is called, and the rest 3829 * of rx page can be reused by driver. 3830 * Or 3831 * 2. Rx page is only owned by the driver when page_offset 3832 * is non-zero, which means page_offset @ truesize will 3833 * be used by stack after skb_add_rx_frag() is called, 3834 * and 0 @ truesize can be reused by driver. 3835 */ 3836 if ((!desc_cb->page_offset && reused) || 3837 ((desc_cb->page_offset + truesize + truesize) <= 3838 hns3_page_size(ring) && desc_cb->page_offset)) { 3839 desc_cb->page_offset += truesize; 3840 desc_cb->reuse_flag = 1; 3841 } else if (desc_cb->page_offset && reused) { 3842 desc_cb->page_offset = 0; 3843 desc_cb->reuse_flag = 1; 3844 } else if (frag_size <= ring->rx_copybreak) { 3845 ret = hns3_handle_rx_copybreak(skb, i, ring, pull_len, desc_cb); 3846 if (!ret) 3847 return; 3848 } 3849 3850out: 3851 desc_cb->pagecnt_bias--; 3852 3853 if (unlikely(!desc_cb->pagecnt_bias)) { 3854 page_ref_add(desc_cb->priv, USHRT_MAX); 3855 desc_cb->pagecnt_bias = USHRT_MAX; 3856 } 3857 3858 skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset, 3859 frag_size, truesize); 3860 3861 if (unlikely(!desc_cb->reuse_flag)) 3862 __page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias); 3863} 3864 3865static int hns3_gro_complete(struct sk_buff *skb, u32 l234info) 3866{ 3867 __be16 type = skb->protocol; 3868 struct tcphdr *th; 3869 u32 depth = 0; 3870 3871 while (eth_type_vlan(type)) { 3872 struct vlan_hdr *vh; 3873 3874 if ((depth + VLAN_HLEN) > skb_headlen(skb)) 3875 return -EFAULT; 3876 3877 vh = (struct vlan_hdr *)(skb->data + depth); 3878 type = vh->h_vlan_encapsulated_proto; 3879 depth += VLAN_HLEN; 3880 } 3881 3882 skb_set_network_header(skb, depth); 3883 3884 if (type == htons(ETH_P_IP)) { 3885 const struct iphdr *iph = ip_hdr(skb); 3886 3887 depth += sizeof(struct iphdr); 3888 skb_set_transport_header(skb, depth); 3889 th = tcp_hdr(skb); 3890 th->check = ~tcp_v4_check(skb->len - depth, iph->saddr, 3891 iph->daddr, 0); 3892 } else if (type == htons(ETH_P_IPV6)) { 3893 const struct ipv6hdr *iph = ipv6_hdr(skb); 3894 3895 depth += sizeof(struct ipv6hdr); 3896 skb_set_transport_header(skb, depth); 3897 th = tcp_hdr(skb); 3898 th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr, 3899 &iph->daddr, 0); 3900 } else { 3901 hns3_rl_err(skb->dev, 3902 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n", 3903 be16_to_cpu(type), depth); 3904 return -EFAULT; 3905 } 3906 3907 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; 3908 if (th->cwr) 3909 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; 3910 3911 if (l234info & BIT(HNS3_RXD_GRO_FIXID_B)) 3912 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID; 3913 3914 skb->csum_start = (unsigned char *)th - skb->head; 3915 skb->csum_offset = offsetof(struct tcphdr, check); 3916 skb->ip_summed = CHECKSUM_PARTIAL; 3917 3918 trace_hns3_gro(skb); 3919 3920 return 0; 3921} 3922 3923static void hns3_checksum_complete(struct hns3_enet_ring *ring, 3924 struct sk_buff *skb, u32 ptype, u16 csum) 3925{ 3926 if (ptype == HNS3_INVALID_PTYPE || 3927 hns3_rx_ptype_tbl[ptype].ip_summed != CHECKSUM_COMPLETE) 3928 return; 3929 3930 hns3_ring_stats_update(ring, csum_complete); 3931 skb->ip_summed = CHECKSUM_COMPLETE; 3932 skb->csum = csum_unfold((__force __sum16)csum); 3933} 3934 3935static void hns3_rx_handle_csum(struct sk_buff *skb, u32 l234info, 3936 u32 ol_info, u32 ptype) 3937{ 3938 int l3_type, l4_type; 3939 int ol4_type; 3940 3941 if (ptype != HNS3_INVALID_PTYPE) { 3942 skb->csum_level = hns3_rx_ptype_tbl[ptype].csum_level; 3943 skb->ip_summed = hns3_rx_ptype_tbl[ptype].ip_summed; 3944 3945 return; 3946 } 3947 3948 ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M, 3949 HNS3_RXD_OL4ID_S); 3950 switch (ol4_type) { 3951 case HNS3_OL4_TYPE_MAC_IN_UDP: 3952 case HNS3_OL4_TYPE_NVGRE: 3953 skb->csum_level = 1; 3954 fallthrough; 3955 case HNS3_OL4_TYPE_NO_TUN: 3956 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, 3957 HNS3_RXD_L3ID_S); 3958 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M, 3959 HNS3_RXD_L4ID_S); 3960 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */ 3961 if ((l3_type == HNS3_L3_TYPE_IPV4 || 3962 l3_type == HNS3_L3_TYPE_IPV6) && 3963 (l4_type == HNS3_L4_TYPE_UDP || 3964 l4_type == HNS3_L4_TYPE_TCP || 3965 l4_type == HNS3_L4_TYPE_SCTP)) 3966 skb->ip_summed = CHECKSUM_UNNECESSARY; 3967 break; 3968 default: 3969 break; 3970 } 3971} 3972 3973static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, 3974 u32 l234info, u32 bd_base_info, u32 ol_info, 3975 u16 csum) 3976{ 3977 struct net_device *netdev = ring_to_netdev(ring); 3978 struct hns3_nic_priv *priv = netdev_priv(netdev); 3979 u32 ptype = HNS3_INVALID_PTYPE; 3980 3981 skb->ip_summed = CHECKSUM_NONE; 3982 3983 skb_checksum_none_assert(skb); 3984 3985 if (!(netdev->features & NETIF_F_RXCSUM)) 3986 return; 3987 3988 if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) 3989 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M, 3990 HNS3_RXD_PTYPE_S); 3991 3992 hns3_checksum_complete(ring, skb, ptype, csum); 3993 3994 /* check if hardware has done checksum */ 3995 if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B))) 3996 return; 3997 3998 if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) | 3999 BIT(HNS3_RXD_OL3E_B) | 4000 BIT(HNS3_RXD_OL4E_B)))) { 4001 skb->ip_summed = CHECKSUM_NONE; 4002 hns3_ring_stats_update(ring, l3l4_csum_err); 4003 4004 return; 4005 } 4006 4007 hns3_rx_handle_csum(skb, l234info, ol_info, ptype); 4008} 4009 4010static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb) 4011{ 4012 if (skb_has_frag_list(skb)) 4013 napi_gro_flush(&ring->tqp_vector->napi, false); 4014 4015 napi_gro_receive(&ring->tqp_vector->napi, skb); 4016} 4017 4018static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring, 4019 struct hns3_desc *desc, u32 l234info, 4020 u16 *vlan_tag) 4021{ 4022 struct hnae3_handle *handle = ring->tqp->handle; 4023 struct pci_dev *pdev = ring->tqp->handle->pdev; 4024 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 4025 4026 if (unlikely(ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2)) { 4027 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag); 4028 if (!(*vlan_tag & VLAN_VID_MASK)) 4029 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag); 4030 4031 return (*vlan_tag != 0); 4032 } 4033 4034#define HNS3_STRP_OUTER_VLAN 0x1 4035#define HNS3_STRP_INNER_VLAN 0x2 4036#define HNS3_STRP_BOTH 0x3 4037 4038 /* Hardware always insert VLAN tag into RX descriptor when 4039 * remove the tag from packet, driver needs to determine 4040 * reporting which tag to stack. 4041 */ 4042 switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M, 4043 HNS3_RXD_STRP_TAGP_S)) { 4044 case HNS3_STRP_OUTER_VLAN: 4045 if (handle->port_base_vlan_state != 4046 HNAE3_PORT_BASE_VLAN_DISABLE) 4047 return false; 4048 4049 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag); 4050 return true; 4051 case HNS3_STRP_INNER_VLAN: 4052 if (handle->port_base_vlan_state != 4053 HNAE3_PORT_BASE_VLAN_DISABLE) 4054 return false; 4055 4056 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag); 4057 return true; 4058 case HNS3_STRP_BOTH: 4059 if (handle->port_base_vlan_state == 4060 HNAE3_PORT_BASE_VLAN_DISABLE) 4061 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag); 4062 else 4063 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag); 4064 4065 return true; 4066 default: 4067 return false; 4068 } 4069} 4070 4071static void hns3_rx_ring_move_fw(struct hns3_enet_ring *ring) 4072{ 4073 ring->desc[ring->next_to_clean].rx.bd_base_info &= 4074 cpu_to_le32(~BIT(HNS3_RXD_VLD_B)); 4075 ring->desc_cb[ring->next_to_clean].refill = 0; 4076 ring->next_to_clean += 1; 4077 4078 if (unlikely(ring->next_to_clean == ring->desc_num)) 4079 ring->next_to_clean = 0; 4080} 4081 4082static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, 4083 unsigned char *va) 4084{ 4085 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean]; 4086 struct net_device *netdev = ring_to_netdev(ring); 4087 struct sk_buff *skb; 4088 4089 ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE); 4090 skb = ring->skb; 4091 if (unlikely(!skb)) { 4092 hns3_rl_err(netdev, "alloc rx skb fail\n"); 4093 hns3_ring_stats_update(ring, sw_err_cnt); 4094 4095 return -ENOMEM; 4096 } 4097 4098 trace_hns3_rx_desc(ring); 4099 prefetchw(skb->data); 4100 4101 ring->pending_buf = 1; 4102 ring->frag_num = 0; 4103 ring->tail_skb = NULL; 4104 if (length <= HNS3_RX_HEAD_SIZE) { 4105 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long))); 4106 4107 /* We can reuse buffer as-is, just make sure it is reusable */ 4108 if (dev_page_is_reusable(desc_cb->priv)) 4109 desc_cb->reuse_flag = 1; 4110 else if (desc_cb->type & DESC_TYPE_PP_FRAG) 4111 page_pool_put_full_page(ring->page_pool, desc_cb->priv, 4112 false); 4113 else /* This page cannot be reused so discard it */ 4114 __page_frag_cache_drain(desc_cb->priv, 4115 desc_cb->pagecnt_bias); 4116 4117 hns3_rx_ring_move_fw(ring); 4118 return 0; 4119 } 4120 4121 if (ring->page_pool) 4122 skb_mark_for_recycle(skb); 4123 4124 hns3_ring_stats_update(ring, seg_pkt_cnt); 4125 4126 ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE); 4127 __skb_put(skb, ring->pull_len); 4128 hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len, 4129 desc_cb); 4130 hns3_rx_ring_move_fw(ring); 4131 4132 return 0; 4133} 4134 4135static int hns3_add_frag(struct hns3_enet_ring *ring) 4136{ 4137 struct sk_buff *skb = ring->skb; 4138 struct sk_buff *head_skb = skb; 4139 struct sk_buff *new_skb; 4140 struct hns3_desc_cb *desc_cb; 4141 struct hns3_desc *desc; 4142 u32 bd_base_info; 4143 4144 do { 4145 desc = &ring->desc[ring->next_to_clean]; 4146 desc_cb = &ring->desc_cb[ring->next_to_clean]; 4147 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 4148 /* make sure HW write desc complete */ 4149 dma_rmb(); 4150 if (!(bd_base_info & BIT(HNS3_RXD_VLD_B))) 4151 return -ENXIO; 4152 4153 if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) { 4154 new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0); 4155 if (unlikely(!new_skb)) { 4156 hns3_rl_err(ring_to_netdev(ring), 4157 "alloc rx fraglist skb fail\n"); 4158 return -ENXIO; 4159 } 4160 4161 if (ring->page_pool) 4162 skb_mark_for_recycle(new_skb); 4163 4164 ring->frag_num = 0; 4165 4166 if (ring->tail_skb) { 4167 ring->tail_skb->next = new_skb; 4168 ring->tail_skb = new_skb; 4169 } else { 4170 skb_shinfo(skb)->frag_list = new_skb; 4171 ring->tail_skb = new_skb; 4172 } 4173 } 4174 4175 if (ring->tail_skb) { 4176 head_skb->truesize += hns3_buf_size(ring); 4177 head_skb->data_len += le16_to_cpu(desc->rx.size); 4178 head_skb->len += le16_to_cpu(desc->rx.size); 4179 skb = ring->tail_skb; 4180 } 4181 4182 dma_sync_single_for_cpu(ring_to_dev(ring), 4183 desc_cb->dma + desc_cb->page_offset, 4184 hns3_buf_size(ring), 4185 DMA_FROM_DEVICE); 4186 4187 hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb); 4188 trace_hns3_rx_desc(ring); 4189 hns3_rx_ring_move_fw(ring); 4190 ring->pending_buf++; 4191 } while (!(bd_base_info & BIT(HNS3_RXD_FE_B))); 4192 4193 return 0; 4194} 4195 4196static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring, 4197 struct sk_buff *skb, u32 l234info, 4198 u32 bd_base_info, u32 ol_info, u16 csum) 4199{ 4200 struct net_device *netdev = ring_to_netdev(ring); 4201 struct hns3_nic_priv *priv = netdev_priv(netdev); 4202 u32 l3_type; 4203 4204 skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info, 4205 HNS3_RXD_GRO_SIZE_M, 4206 HNS3_RXD_GRO_SIZE_S); 4207 /* if there is no HW GRO, do not set gro params */ 4208 if (!skb_shinfo(skb)->gso_size) { 4209 hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info, 4210 csum); 4211 return 0; 4212 } 4213 4214 NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info, 4215 HNS3_RXD_GRO_COUNT_M, 4216 HNS3_RXD_GRO_COUNT_S); 4217 4218 if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) { 4219 u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M, 4220 HNS3_RXD_PTYPE_S); 4221 4222 l3_type = hns3_rx_ptype_tbl[ptype].l3_type; 4223 } else { 4224 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, 4225 HNS3_RXD_L3ID_S); 4226 } 4227 4228 if (l3_type == HNS3_L3_TYPE_IPV4) 4229 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; 4230 else if (l3_type == HNS3_L3_TYPE_IPV6) 4231 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; 4232 else 4233 return -EFAULT; 4234 4235 return hns3_gro_complete(skb, l234info); 4236} 4237 4238static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring, 4239 struct sk_buff *skb, u32 rss_hash, 4240 u32 l234info, u32 ol_info) 4241{ 4242 enum pkt_hash_types rss_type = PKT_HASH_TYPE_NONE; 4243 struct net_device *netdev = ring_to_netdev(ring); 4244 struct hns3_nic_priv *priv = netdev_priv(netdev); 4245 4246 if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) { 4247 u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M, 4248 HNS3_RXD_PTYPE_S); 4249 4250 rss_type = hns3_rx_ptype_tbl[ptype].hash_type; 4251 } else { 4252 int l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M, 4253 HNS3_RXD_L3ID_S); 4254 int l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M, 4255 HNS3_RXD_L4ID_S); 4256 4257 if (l3_type == HNS3_L3_TYPE_IPV4 || 4258 l3_type == HNS3_L3_TYPE_IPV6) { 4259 if (l4_type == HNS3_L4_TYPE_UDP || 4260 l4_type == HNS3_L4_TYPE_TCP || 4261 l4_type == HNS3_L4_TYPE_SCTP) 4262 rss_type = PKT_HASH_TYPE_L4; 4263 else if (l4_type == HNS3_L4_TYPE_IGMP || 4264 l4_type == HNS3_L4_TYPE_ICMP) 4265 rss_type = PKT_HASH_TYPE_L3; 4266 } 4267 } 4268 4269 skb_set_hash(skb, rss_hash, rss_type); 4270} 4271 4272static void hns3_handle_rx_ts_info(struct net_device *netdev, 4273 struct hns3_desc *desc, struct sk_buff *skb, 4274 u32 bd_base_info) 4275{ 4276 if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) { 4277 struct hnae3_handle *h = hns3_get_handle(netdev); 4278 u32 nsec = le32_to_cpu(desc->ts_nsec); 4279 u32 sec = le32_to_cpu(desc->ts_sec); 4280 4281 if (h->ae_algo->ops->get_rx_hwts) 4282 h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec); 4283 } 4284} 4285 4286static void hns3_handle_rx_vlan_tag(struct hns3_enet_ring *ring, 4287 struct hns3_desc *desc, struct sk_buff *skb, 4288 u32 l234info) 4289{ 4290 struct net_device *netdev = ring_to_netdev(ring); 4291 4292 /* Based on hw strategy, the tag offloaded will be stored at 4293 * ot_vlan_tag in two layer tag case, and stored at vlan_tag 4294 * in one layer tag case. 4295 */ 4296 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) { 4297 u16 vlan_tag; 4298 4299 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag)) 4300 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 4301 vlan_tag); 4302 } 4303} 4304 4305static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) 4306{ 4307 struct net_device *netdev = ring_to_netdev(ring); 4308 enum hns3_pkt_l2t_type l2_frame_type; 4309 u32 bd_base_info, l234info, ol_info; 4310 struct hns3_desc *desc; 4311 unsigned int len; 4312 int pre_ntc, ret; 4313 u16 csum; 4314 4315 /* bdinfo handled below is only valid on the last BD of the 4316 * current packet, and ring->next_to_clean indicates the first 4317 * descriptor of next packet, so need - 1 below. 4318 */ 4319 pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) : 4320 (ring->desc_num - 1); 4321 desc = &ring->desc[pre_ntc]; 4322 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 4323 l234info = le32_to_cpu(desc->rx.l234_info); 4324 ol_info = le32_to_cpu(desc->rx.ol_info); 4325 csum = le16_to_cpu(desc->csum); 4326 4327 hns3_handle_rx_ts_info(netdev, desc, skb, bd_base_info); 4328 4329 hns3_handle_rx_vlan_tag(ring, desc, skb, l234info); 4330 4331 if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) | 4332 BIT(HNS3_RXD_L2E_B))))) { 4333 u64_stats_update_begin(&ring->syncp); 4334 if (l234info & BIT(HNS3_RXD_L2E_B)) 4335 ring->stats.l2_err++; 4336 else 4337 ring->stats.err_pkt_len++; 4338 u64_stats_update_end(&ring->syncp); 4339 4340 return -EFAULT; 4341 } 4342 4343 len = skb->len; 4344 4345 /* Do update ip stack process */ 4346 skb->protocol = eth_type_trans(skb, netdev); 4347 4348 /* This is needed in order to enable forwarding support */ 4349 ret = hns3_set_gro_and_checksum(ring, skb, l234info, 4350 bd_base_info, ol_info, csum); 4351 if (unlikely(ret)) { 4352 hns3_ring_stats_update(ring, rx_err_cnt); 4353 return ret; 4354 } 4355 4356 l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M, 4357 HNS3_RXD_DMAC_S); 4358 4359 u64_stats_update_begin(&ring->syncp); 4360 ring->stats.rx_pkts++; 4361 ring->stats.rx_bytes += len; 4362 4363 if (l2_frame_type == HNS3_L2_TYPE_MULTICAST) 4364 ring->stats.rx_multicast++; 4365 4366 u64_stats_update_end(&ring->syncp); 4367 4368 ring->tqp_vector->rx_group.total_bytes += len; 4369 4370 hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash), 4371 l234info, ol_info); 4372 return 0; 4373} 4374 4375static int hns3_handle_rx_bd(struct hns3_enet_ring *ring) 4376{ 4377 struct sk_buff *skb = ring->skb; 4378 struct hns3_desc_cb *desc_cb; 4379 struct hns3_desc *desc; 4380 unsigned int length; 4381 u32 bd_base_info; 4382 int ret; 4383 4384 desc = &ring->desc[ring->next_to_clean]; 4385 desc_cb = &ring->desc_cb[ring->next_to_clean]; 4386 4387 prefetch(desc); 4388 4389 if (!skb) { 4390 bd_base_info = le32_to_cpu(desc->rx.bd_base_info); 4391 /* Check valid BD */ 4392 if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B)))) 4393 return -ENXIO; 4394 4395 dma_rmb(); 4396 length = le16_to_cpu(desc->rx.size); 4397 4398 ring->va = desc_cb->buf + desc_cb->page_offset; 4399 4400 dma_sync_single_for_cpu(ring_to_dev(ring), 4401 desc_cb->dma + desc_cb->page_offset, 4402 hns3_buf_size(ring), 4403 DMA_FROM_DEVICE); 4404 4405 /* Prefetch first cache line of first page. 4406 * Idea is to cache few bytes of the header of the packet. 4407 * Our L1 Cache line size is 64B so need to prefetch twice to make 4408 * it 128B. But in actual we can have greater size of caches with 4409 * 128B Level 1 cache lines. In such a case, single fetch would 4410 * suffice to cache in the relevant part of the header. 4411 */ 4412 net_prefetch(ring->va); 4413 4414 ret = hns3_alloc_skb(ring, length, ring->va); 4415 skb = ring->skb; 4416 4417 if (ret < 0) /* alloc buffer fail */ 4418 return ret; 4419 if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */ 4420 ret = hns3_add_frag(ring); 4421 if (ret) 4422 return ret; 4423 } 4424 } else { 4425 ret = hns3_add_frag(ring); 4426 if (ret) 4427 return ret; 4428 } 4429 4430 /* As the head data may be changed when GRO enable, copy 4431 * the head data in after other data rx completed 4432 */ 4433 if (skb->len > HNS3_RX_HEAD_SIZE) 4434 memcpy(skb->data, ring->va, 4435 ALIGN(ring->pull_len, sizeof(long))); 4436 4437 ret = hns3_handle_bdinfo(ring, skb); 4438 if (unlikely(ret)) { 4439 dev_kfree_skb_any(skb); 4440 return ret; 4441 } 4442 4443 skb_record_rx_queue(skb, ring->tqp->tqp_index); 4444 return 0; 4445} 4446 4447int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget, 4448 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *)) 4449{ 4450#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16 4451 int unused_count = hns3_desc_unused(ring); 4452 bool failure = false; 4453 int recv_pkts = 0; 4454 int err; 4455 4456 unused_count -= ring->pending_buf; 4457 4458 while (recv_pkts < budget) { 4459 /* Reuse or realloc buffers */ 4460 if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) { 4461 failure = failure || 4462 hns3_nic_alloc_rx_buffers(ring, unused_count); 4463 unused_count = 0; 4464 } 4465 4466 /* Poll one pkt */ 4467 err = hns3_handle_rx_bd(ring); 4468 /* Do not get FE for the packet or failed to alloc skb */ 4469 if (unlikely(!ring->skb || err == -ENXIO)) { 4470 goto out; 4471 } else if (likely(!err)) { 4472 rx_fn(ring, ring->skb); 4473 recv_pkts++; 4474 } 4475 4476 unused_count += ring->pending_buf; 4477 ring->skb = NULL; 4478 ring->pending_buf = 0; 4479 } 4480 4481out: 4482 /* sync head pointer before exiting, since hardware will calculate 4483 * FBD number with head pointer 4484 */ 4485 if (unused_count > 0) 4486 failure = failure || 4487 hns3_nic_alloc_rx_buffers(ring, unused_count); 4488 4489 return failure ? budget : recv_pkts; 4490} 4491 4492static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector) 4493{ 4494 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group; 4495 struct dim_sample sample = {}; 4496 4497 if (!rx_group->coal.adapt_enable) 4498 return; 4499 4500 dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets, 4501 rx_group->total_bytes, &sample); 4502 net_dim(&rx_group->dim, &sample); 4503} 4504 4505static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector) 4506{ 4507 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group; 4508 struct dim_sample sample = {}; 4509 4510 if (!tx_group->coal.adapt_enable) 4511 return; 4512 4513 dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets, 4514 tx_group->total_bytes, &sample); 4515 net_dim(&tx_group->dim, &sample); 4516} 4517 4518static int hns3_nic_common_poll(struct napi_struct *napi, int budget) 4519{ 4520 struct hns3_nic_priv *priv = netdev_priv(napi->dev); 4521 struct hns3_enet_ring *ring; 4522 int rx_pkt_total = 0; 4523 4524 struct hns3_enet_tqp_vector *tqp_vector = 4525 container_of(napi, struct hns3_enet_tqp_vector, napi); 4526 bool clean_complete = true; 4527 int rx_budget = budget; 4528 4529 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) { 4530 napi_complete(napi); 4531 return 0; 4532 } 4533 4534 /* Since the actual Tx work is minimal, we can give the Tx a larger 4535 * budget and be more aggressive about cleaning up the Tx descriptors. 4536 */ 4537 hns3_for_each_ring(ring, tqp_vector->tx_group) 4538 hns3_clean_tx_ring(ring, budget); 4539 4540 /* make sure rx ring budget not smaller than 1 */ 4541 if (tqp_vector->num_tqps > 1) 4542 rx_budget = max(budget / tqp_vector->num_tqps, 1); 4543 4544 hns3_for_each_ring(ring, tqp_vector->rx_group) { 4545 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget, 4546 hns3_rx_skb); 4547 if (rx_cleaned >= rx_budget) 4548 clean_complete = false; 4549 4550 rx_pkt_total += rx_cleaned; 4551 } 4552 4553 tqp_vector->rx_group.total_packets += rx_pkt_total; 4554 4555 if (!clean_complete) 4556 return budget; 4557 4558 if (napi_complete(napi) && 4559 likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) { 4560 hns3_update_rx_int_coalesce(tqp_vector); 4561 hns3_update_tx_int_coalesce(tqp_vector); 4562 4563 hns3_mask_vector_irq(tqp_vector, 1); 4564 } 4565 4566 return rx_pkt_total; 4567} 4568 4569static int hns3_create_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, 4570 struct hnae3_ring_chain_node **head, 4571 bool is_tx) 4572{ 4573 u32 bit_value = is_tx ? HNAE3_RING_TYPE_TX : HNAE3_RING_TYPE_RX; 4574 u32 field_value = is_tx ? HNAE3_RING_GL_TX : HNAE3_RING_GL_RX; 4575 struct hnae3_ring_chain_node *cur_chain = *head; 4576 struct pci_dev *pdev = tqp_vector->handle->pdev; 4577 struct hnae3_ring_chain_node *chain; 4578 struct hns3_enet_ring *ring; 4579 4580 ring = is_tx ? tqp_vector->tx_group.ring : tqp_vector->rx_group.ring; 4581 4582 if (cur_chain) { 4583 while (cur_chain->next) 4584 cur_chain = cur_chain->next; 4585 } 4586 4587 while (ring) { 4588 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); 4589 if (!chain) 4590 return -ENOMEM; 4591 if (cur_chain) 4592 cur_chain->next = chain; 4593 else 4594 *head = chain; 4595 chain->tqp_index = ring->tqp->tqp_index; 4596 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B, 4597 bit_value); 4598 hnae3_set_field(chain->int_gl_idx, 4599 HNAE3_RING_GL_IDX_M, 4600 HNAE3_RING_GL_IDX_S, field_value); 4601 4602 cur_chain = chain; 4603 4604 ring = ring->next; 4605 } 4606 4607 return 0; 4608} 4609 4610static struct hnae3_ring_chain_node * 4611hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector) 4612{ 4613 struct pci_dev *pdev = tqp_vector->handle->pdev; 4614 struct hnae3_ring_chain_node *cur_chain = NULL; 4615 struct hnae3_ring_chain_node *chain; 4616 4617 if (hns3_create_ring_chain(tqp_vector, &cur_chain, true)) 4618 goto err_free_chain; 4619 4620 if (hns3_create_ring_chain(tqp_vector, &cur_chain, false)) 4621 goto err_free_chain; 4622 4623 return cur_chain; 4624 4625err_free_chain: 4626 while (cur_chain) { 4627 chain = cur_chain->next; 4628 devm_kfree(&pdev->dev, cur_chain); 4629 cur_chain = chain; 4630 } 4631 4632 return NULL; 4633} 4634 4635static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, 4636 struct hnae3_ring_chain_node *head) 4637{ 4638 struct pci_dev *pdev = tqp_vector->handle->pdev; 4639 struct hnae3_ring_chain_node *chain_tmp, *chain; 4640 4641 chain = head; 4642 4643 while (chain) { 4644 chain_tmp = chain->next; 4645 devm_kfree(&pdev->dev, chain); 4646 chain = chain_tmp; 4647 } 4648} 4649 4650static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group, 4651 struct hns3_enet_ring *ring) 4652{ 4653 ring->next = group->ring; 4654 group->ring = ring; 4655 4656 group->count++; 4657} 4658 4659static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv) 4660{ 4661 struct pci_dev *pdev = priv->ae_handle->pdev; 4662 struct hns3_enet_tqp_vector *tqp_vector; 4663 int num_vectors = priv->vector_num; 4664 int numa_node; 4665 int vector_i; 4666 4667 numa_node = dev_to_node(&pdev->dev); 4668 4669 for (vector_i = 0; vector_i < num_vectors; vector_i++) { 4670 tqp_vector = &priv->tqp_vector[vector_i]; 4671 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node), 4672 &tqp_vector->affinity_mask); 4673 } 4674} 4675 4676static void hns3_rx_dim_work(struct work_struct *work) 4677{ 4678 struct dim *dim = container_of(work, struct dim, work); 4679 struct hns3_enet_ring_group *group = container_of(dim, 4680 struct hns3_enet_ring_group, dim); 4681 struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector; 4682 struct dim_cq_moder cur_moder = 4683 net_dim_get_rx_moderation(dim->mode, dim->profile_ix); 4684 4685 hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec); 4686 tqp_vector->rx_group.coal.int_gl = cur_moder.usec; 4687 4688 if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) { 4689 hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts); 4690 tqp_vector->rx_group.coal.int_ql = cur_moder.pkts; 4691 } 4692 4693 dim->state = DIM_START_MEASURE; 4694} 4695 4696static void hns3_tx_dim_work(struct work_struct *work) 4697{ 4698 struct dim *dim = container_of(work, struct dim, work); 4699 struct hns3_enet_ring_group *group = container_of(dim, 4700 struct hns3_enet_ring_group, dim); 4701 struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector; 4702 struct dim_cq_moder cur_moder = 4703 net_dim_get_tx_moderation(dim->mode, dim->profile_ix); 4704 4705 hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec); 4706 tqp_vector->tx_group.coal.int_gl = cur_moder.usec; 4707 4708 if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) { 4709 hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts); 4710 tqp_vector->tx_group.coal.int_ql = cur_moder.pkts; 4711 } 4712 4713 dim->state = DIM_START_MEASURE; 4714} 4715 4716static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector) 4717{ 4718 INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work); 4719 INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work); 4720} 4721 4722static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) 4723{ 4724 struct hnae3_handle *h = priv->ae_handle; 4725 struct hns3_enet_tqp_vector *tqp_vector; 4726 int ret; 4727 int i; 4728 4729 hns3_nic_set_cpumask(priv); 4730 4731 for (i = 0; i < priv->vector_num; i++) { 4732 tqp_vector = &priv->tqp_vector[i]; 4733 hns3_vector_coalesce_init_hw(tqp_vector, priv); 4734 tqp_vector->num_tqps = 0; 4735 hns3_nic_init_dim(tqp_vector); 4736 } 4737 4738 for (i = 0; i < h->kinfo.num_tqps; i++) { 4739 u16 vector_i = i % priv->vector_num; 4740 u16 tqp_num = h->kinfo.num_tqps; 4741 4742 tqp_vector = &priv->tqp_vector[vector_i]; 4743 4744 hns3_add_ring_to_group(&tqp_vector->tx_group, 4745 &priv->ring[i]); 4746 4747 hns3_add_ring_to_group(&tqp_vector->rx_group, 4748 &priv->ring[i + tqp_num]); 4749 4750 priv->ring[i].tqp_vector = tqp_vector; 4751 priv->ring[i + tqp_num].tqp_vector = tqp_vector; 4752 tqp_vector->num_tqps++; 4753 } 4754 4755 for (i = 0; i < priv->vector_num; i++) { 4756 struct hnae3_ring_chain_node *vector_ring_chain; 4757 4758 tqp_vector = &priv->tqp_vector[i]; 4759 4760 tqp_vector->rx_group.total_bytes = 0; 4761 tqp_vector->rx_group.total_packets = 0; 4762 tqp_vector->tx_group.total_bytes = 0; 4763 tqp_vector->tx_group.total_packets = 0; 4764 tqp_vector->handle = h; 4765 4766 vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector); 4767 if (!vector_ring_chain) { 4768 ret = -ENOMEM; 4769 goto map_ring_fail; 4770 } 4771 4772 ret = h->ae_algo->ops->map_ring_to_vector(h, 4773 tqp_vector->vector_irq, vector_ring_chain); 4774 4775 hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain); 4776 4777 if (ret) 4778 goto map_ring_fail; 4779 4780 netif_napi_add(priv->netdev, &tqp_vector->napi, 4781 hns3_nic_common_poll); 4782 } 4783 4784 return 0; 4785 4786map_ring_fail: 4787 while (i--) 4788 netif_napi_del(&priv->tqp_vector[i].napi); 4789 4790 return ret; 4791} 4792 4793static void hns3_nic_init_coal_cfg(struct hns3_nic_priv *priv) 4794{ 4795 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); 4796 struct hns3_enet_coalesce *tx_coal = &priv->tx_coal; 4797 struct hns3_enet_coalesce *rx_coal = &priv->rx_coal; 4798 4799 /* initialize the configuration for interrupt coalescing. 4800 * 1. GL (Interrupt Gap Limiter) 4801 * 2. RL (Interrupt Rate Limiter) 4802 * 3. QL (Interrupt Quantity Limiter) 4803 * 4804 * Default: enable interrupt coalescing self-adaptive and GL 4805 */ 4806 tx_coal->adapt_enable = 1; 4807 rx_coal->adapt_enable = 1; 4808 4809 tx_coal->int_gl = HNS3_INT_GL_50K; 4810 rx_coal->int_gl = HNS3_INT_GL_50K; 4811 4812 rx_coal->flow_level = HNS3_FLOW_LOW; 4813 tx_coal->flow_level = HNS3_FLOW_LOW; 4814 4815 if (ae_dev->dev_specs.int_ql_max) { 4816 tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; 4817 rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG; 4818 } 4819} 4820 4821static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv) 4822{ 4823 struct hnae3_handle *h = priv->ae_handle; 4824 struct hns3_enet_tqp_vector *tqp_vector; 4825 struct hnae3_vector_info *vector; 4826 struct pci_dev *pdev = h->pdev; 4827 u16 tqp_num = h->kinfo.num_tqps; 4828 u16 vector_num; 4829 int ret = 0; 4830 u16 i; 4831 4832 /* RSS size, cpu online and vector_num should be the same */ 4833 /* Should consider 2p/4p later */ 4834 vector_num = min_t(u16, num_online_cpus(), tqp_num); 4835 4836 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector), 4837 GFP_KERNEL); 4838 if (!vector) 4839 return -ENOMEM; 4840 4841 /* save the actual available vector number */ 4842 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector); 4843 4844 priv->vector_num = vector_num; 4845 priv->tqp_vector = (struct hns3_enet_tqp_vector *) 4846 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector), 4847 GFP_KERNEL); 4848 if (!priv->tqp_vector) { 4849 ret = -ENOMEM; 4850 goto out; 4851 } 4852 4853 for (i = 0; i < priv->vector_num; i++) { 4854 tqp_vector = &priv->tqp_vector[i]; 4855 tqp_vector->idx = i; 4856 tqp_vector->mask_addr = vector[i].io_addr; 4857 tqp_vector->vector_irq = vector[i].vector; 4858 hns3_vector_coalesce_init(tqp_vector, priv); 4859 } 4860 4861out: 4862 devm_kfree(&pdev->dev, vector); 4863 return ret; 4864} 4865 4866static void hns3_clear_ring_group(struct hns3_enet_ring_group *group) 4867{ 4868 group->ring = NULL; 4869 group->count = 0; 4870} 4871 4872static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv) 4873{ 4874 struct hnae3_ring_chain_node *vector_ring_chain; 4875 struct hnae3_handle *h = priv->ae_handle; 4876 struct hns3_enet_tqp_vector *tqp_vector; 4877 int i; 4878 4879 for (i = 0; i < priv->vector_num; i++) { 4880 tqp_vector = &priv->tqp_vector[i]; 4881 4882 if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring) 4883 continue; 4884 4885 /* Since the mapping can be overwritten, when fail to get the 4886 * chain between vector and ring, we should go on to deal with 4887 * the remaining options. 4888 */ 4889 vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector); 4890 if (!vector_ring_chain) 4891 dev_warn(priv->dev, "failed to get ring chain\n"); 4892 4893 h->ae_algo->ops->unmap_ring_from_vector(h, 4894 tqp_vector->vector_irq, vector_ring_chain); 4895 4896 hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain); 4897 4898 hns3_clear_ring_group(&tqp_vector->rx_group); 4899 hns3_clear_ring_group(&tqp_vector->tx_group); 4900 netif_napi_del(&priv->tqp_vector[i].napi); 4901 } 4902} 4903 4904static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv) 4905{ 4906 struct hnae3_handle *h = priv->ae_handle; 4907 struct pci_dev *pdev = h->pdev; 4908 int i, ret; 4909 4910 for (i = 0; i < priv->vector_num; i++) { 4911 struct hns3_enet_tqp_vector *tqp_vector; 4912 4913 tqp_vector = &priv->tqp_vector[i]; 4914 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq); 4915 if (ret) 4916 return; 4917 } 4918 4919 devm_kfree(&pdev->dev, priv->tqp_vector); 4920} 4921 4922static void hns3_update_tx_spare_buf_config(struct hns3_nic_priv *priv) 4923{ 4924#define HNS3_MIN_SPARE_BUF_SIZE (2 * 1024 * 1024) 4925#define HNS3_MAX_PACKET_SIZE (64 * 1024) 4926 4927 struct iommu_domain *domain = iommu_get_domain_for_dev(priv->dev); 4928 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); 4929 struct hnae3_handle *handle = priv->ae_handle; 4930 4931 if (ae_dev->dev_version < HNAE3_DEVICE_VERSION_V3) 4932 return; 4933 4934 if (!(domain && iommu_is_dma_domain(domain))) 4935 return; 4936 4937 priv->min_tx_copybreak = HNS3_MAX_PACKET_SIZE; 4938 priv->min_tx_spare_buf_size = HNS3_MIN_SPARE_BUF_SIZE; 4939 4940 if (priv->tx_copybreak < priv->min_tx_copybreak) 4941 priv->tx_copybreak = priv->min_tx_copybreak; 4942 if (handle->kinfo.tx_spare_buf_size < priv->min_tx_spare_buf_size) 4943 handle->kinfo.tx_spare_buf_size = priv->min_tx_spare_buf_size; 4944} 4945 4946static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv, 4947 unsigned int ring_type) 4948{ 4949 int queue_num = priv->ae_handle->kinfo.num_tqps; 4950 struct hns3_enet_ring *ring; 4951 int desc_num; 4952 4953 if (ring_type == HNAE3_RING_TYPE_TX) { 4954 ring = &priv->ring[q->tqp_index]; 4955 desc_num = priv->ae_handle->kinfo.num_tx_desc; 4956 ring->queue_index = q->tqp_index; 4957 ring->tx_copybreak = priv->tx_copybreak; 4958 ring->last_to_use = 0; 4959 } else { 4960 ring = &priv->ring[q->tqp_index + queue_num]; 4961 desc_num = priv->ae_handle->kinfo.num_rx_desc; 4962 ring->queue_index = q->tqp_index; 4963 ring->rx_copybreak = priv->rx_copybreak; 4964 } 4965 4966 hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type); 4967 4968 ring->tqp = q; 4969 ring->desc = NULL; 4970 ring->desc_cb = NULL; 4971 ring->dev = priv->dev; 4972 ring->desc_dma_addr = 0; 4973 ring->buf_size = q->buf_size; 4974 ring->desc_num = desc_num; 4975 ring->next_to_use = 0; 4976 ring->next_to_clean = 0; 4977} 4978 4979static void hns3_queue_to_ring(struct hnae3_queue *tqp, 4980 struct hns3_nic_priv *priv) 4981{ 4982 hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX); 4983 hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX); 4984} 4985 4986static int hns3_get_ring_config(struct hns3_nic_priv *priv) 4987{ 4988 struct hnae3_handle *h = priv->ae_handle; 4989 struct pci_dev *pdev = h->pdev; 4990 int i; 4991 4992 priv->ring = devm_kzalloc(&pdev->dev, 4993 array3_size(h->kinfo.num_tqps, 4994 sizeof(*priv->ring), 2), 4995 GFP_KERNEL); 4996 if (!priv->ring) 4997 return -ENOMEM; 4998 4999 for (i = 0; i < h->kinfo.num_tqps; i++) 5000 hns3_queue_to_ring(h->kinfo.tqp[i], priv); 5001 5002 return 0; 5003} 5004 5005static void hns3_put_ring_config(struct hns3_nic_priv *priv) 5006{ 5007 if (!priv->ring) 5008 return; 5009 5010 devm_kfree(priv->dev, priv->ring); 5011 priv->ring = NULL; 5012} 5013 5014static void hns3_alloc_page_pool(struct hns3_enet_ring *ring) 5015{ 5016 struct page_pool_params pp_params = { 5017 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 5018 .order = hns3_page_order(ring), 5019 .pool_size = ring->desc_num * hns3_buf_size(ring) / 5020 (PAGE_SIZE << hns3_page_order(ring)), 5021 .nid = dev_to_node(ring_to_dev(ring)), 5022 .dev = ring_to_dev(ring), 5023 .dma_dir = DMA_FROM_DEVICE, 5024 .offset = 0, 5025 .max_len = PAGE_SIZE << hns3_page_order(ring), 5026 }; 5027 5028 ring->page_pool = page_pool_create(&pp_params); 5029 if (IS_ERR(ring->page_pool)) { 5030 dev_warn(ring_to_dev(ring), "page pool creation failed: %ld\n", 5031 PTR_ERR(ring->page_pool)); 5032 ring->page_pool = NULL; 5033 } 5034} 5035 5036static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring) 5037{ 5038 int ret; 5039 5040 if (ring->desc_num <= 0 || ring->buf_size <= 0) 5041 return -EINVAL; 5042 5043 ring->desc_cb = devm_kcalloc(ring_to_dev(ring), ring->desc_num, 5044 sizeof(ring->desc_cb[0]), GFP_KERNEL); 5045 if (!ring->desc_cb) { 5046 ret = -ENOMEM; 5047 goto out; 5048 } 5049 5050 ret = hns3_alloc_desc(ring); 5051 if (ret) 5052 goto out_with_desc_cb; 5053 5054 if (!HNAE3_IS_TX_RING(ring)) { 5055 if (page_pool_enabled) 5056 hns3_alloc_page_pool(ring); 5057 5058 ret = hns3_alloc_ring_buffers(ring); 5059 if (ret) 5060 goto out_with_desc; 5061 } else { 5062 hns3_init_tx_spare_buffer(ring); 5063 } 5064 5065 return 0; 5066 5067out_with_desc: 5068 hns3_free_desc(ring); 5069out_with_desc_cb: 5070 devm_kfree(ring_to_dev(ring), ring->desc_cb); 5071 ring->desc_cb = NULL; 5072out: 5073 return ret; 5074} 5075 5076void hns3_fini_ring(struct hns3_enet_ring *ring) 5077{ 5078 hns3_free_desc(ring); 5079 devm_kfree(ring_to_dev(ring), ring->desc_cb); 5080 ring->desc_cb = NULL; 5081 ring->next_to_clean = 0; 5082 ring->next_to_use = 0; 5083 ring->last_to_use = 0; 5084 ring->pending_buf = 0; 5085 if (!HNAE3_IS_TX_RING(ring) && ring->skb) { 5086 dev_kfree_skb_any(ring->skb); 5087 ring->skb = NULL; 5088 } else if (HNAE3_IS_TX_RING(ring) && ring->tx_spare) { 5089 struct hns3_tx_spare *tx_spare = ring->tx_spare; 5090 5091 dma_unmap_page(ring_to_dev(ring), tx_spare->dma, tx_spare->len, 5092 DMA_TO_DEVICE); 5093 free_pages((unsigned long)tx_spare->buf, 5094 get_order(tx_spare->len)); 5095 devm_kfree(ring_to_dev(ring), tx_spare); 5096 ring->tx_spare = NULL; 5097 } 5098 5099 if (!HNAE3_IS_TX_RING(ring) && ring->page_pool) { 5100 page_pool_destroy(ring->page_pool); 5101 ring->page_pool = NULL; 5102 } 5103} 5104 5105static int hns3_buf_size2type(u32 buf_size) 5106{ 5107 int bd_size_type; 5108 5109 switch (buf_size) { 5110 case 512: 5111 bd_size_type = HNS3_BD_SIZE_512_TYPE; 5112 break; 5113 case 1024: 5114 bd_size_type = HNS3_BD_SIZE_1024_TYPE; 5115 break; 5116 case 2048: 5117 bd_size_type = HNS3_BD_SIZE_2048_TYPE; 5118 break; 5119 case 4096: 5120 bd_size_type = HNS3_BD_SIZE_4096_TYPE; 5121 break; 5122 default: 5123 bd_size_type = HNS3_BD_SIZE_2048_TYPE; 5124 } 5125 5126 return bd_size_type; 5127} 5128 5129static void hns3_init_ring_hw(struct hns3_enet_ring *ring) 5130{ 5131 dma_addr_t dma = ring->desc_dma_addr; 5132 struct hnae3_queue *q = ring->tqp; 5133 5134 if (!HNAE3_IS_TX_RING(ring)) { 5135 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, (u32)dma); 5136 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG, 5137 (u32)((dma >> 31) >> 1)); 5138 5139 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG, 5140 hns3_buf_size2type(ring->buf_size)); 5141 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG, 5142 ring->desc_num / 8 - 1); 5143 } else { 5144 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG, 5145 (u32)dma); 5146 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG, 5147 (u32)((dma >> 31) >> 1)); 5148 5149 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG, 5150 ring->desc_num / 8 - 1); 5151 } 5152} 5153 5154static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv) 5155{ 5156 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo; 5157 struct hnae3_tc_info *tc_info = &kinfo->tc_info; 5158 int i; 5159 5160 for (i = 0; i < tc_info->num_tc; i++) { 5161 int j; 5162 5163 for (j = 0; j < tc_info->tqp_count[i]; j++) { 5164 struct hnae3_queue *q; 5165 5166 q = priv->ring[tc_info->tqp_offset[i] + j].tqp; 5167 hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG, i); 5168 } 5169 } 5170} 5171 5172int hns3_init_all_ring(struct hns3_nic_priv *priv) 5173{ 5174 struct hnae3_handle *h = priv->ae_handle; 5175 int ring_num = h->kinfo.num_tqps * 2; 5176 int i, j; 5177 int ret; 5178 5179 hns3_update_tx_spare_buf_config(priv); 5180 for (i = 0; i < ring_num; i++) { 5181 ret = hns3_alloc_ring_memory(&priv->ring[i]); 5182 if (ret) { 5183 dev_err(priv->dev, 5184 "Alloc ring memory fail! ret=%d\n", ret); 5185 goto out_when_alloc_ring_memory; 5186 } 5187 5188 u64_stats_init(&priv->ring[i].syncp); 5189 cond_resched(); 5190 } 5191 5192 return 0; 5193 5194out_when_alloc_ring_memory: 5195 for (j = i - 1; j >= 0; j--) 5196 hns3_fini_ring(&priv->ring[j]); 5197 5198 return -ENOMEM; 5199} 5200 5201static void hns3_uninit_all_ring(struct hns3_nic_priv *priv) 5202{ 5203 struct hnae3_handle *h = priv->ae_handle; 5204 int i; 5205 5206 for (i = 0; i < h->kinfo.num_tqps; i++) { 5207 hns3_fini_ring(&priv->ring[i]); 5208 hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]); 5209 } 5210} 5211 5212/* Set mac addr if it is configured. or leave it to the AE driver */ 5213static int hns3_init_mac_addr(struct net_device *netdev) 5214{ 5215 struct hns3_nic_priv *priv = netdev_priv(netdev); 5216 char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; 5217 struct hnae3_handle *h = priv->ae_handle; 5218 u8 mac_addr_temp[ETH_ALEN] = {0}; 5219 int ret = 0; 5220 5221 if (h->ae_algo->ops->get_mac_addr) 5222 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp); 5223 5224 /* Check if the MAC address is valid, if not get a random one */ 5225 if (!is_valid_ether_addr(mac_addr_temp)) { 5226 eth_hw_addr_random(netdev); 5227 hnae3_format_mac_addr(format_mac_addr, netdev->dev_addr); 5228 dev_warn(priv->dev, "using random MAC address %s\n", 5229 format_mac_addr); 5230 } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) { 5231 eth_hw_addr_set(netdev, mac_addr_temp); 5232 ether_addr_copy(netdev->perm_addr, mac_addr_temp); 5233 } else { 5234 return 0; 5235 } 5236 5237 if (h->ae_algo->ops->set_mac_addr) 5238 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true); 5239 5240 return ret; 5241} 5242 5243static int hns3_init_phy(struct net_device *netdev) 5244{ 5245 struct hnae3_handle *h = hns3_get_handle(netdev); 5246 int ret = 0; 5247 5248 if (h->ae_algo->ops->mac_connect_phy) 5249 ret = h->ae_algo->ops->mac_connect_phy(h); 5250 5251 return ret; 5252} 5253 5254static void hns3_uninit_phy(struct net_device *netdev) 5255{ 5256 struct hnae3_handle *h = hns3_get_handle(netdev); 5257 5258 if (h->ae_algo->ops->mac_disconnect_phy) 5259 h->ae_algo->ops->mac_disconnect_phy(h); 5260} 5261 5262static int hns3_client_start(struct hnae3_handle *handle) 5263{ 5264 if (!handle->ae_algo->ops->client_start) 5265 return 0; 5266 5267 return handle->ae_algo->ops->client_start(handle); 5268} 5269 5270static void hns3_client_stop(struct hnae3_handle *handle) 5271{ 5272 if (!handle->ae_algo->ops->client_stop) 5273 return; 5274 5275 handle->ae_algo->ops->client_stop(handle); 5276} 5277 5278static void hns3_info_show(struct hns3_nic_priv *priv) 5279{ 5280 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo; 5281 char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; 5282 5283 hnae3_format_mac_addr(format_mac_addr, priv->netdev->dev_addr); 5284 dev_info(priv->dev, "MAC address: %s\n", format_mac_addr); 5285 dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps); 5286 dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size); 5287 dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size); 5288 dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len); 5289 dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc); 5290 dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc); 5291 dev_info(priv->dev, "Total number of enabled TCs: %u\n", 5292 kinfo->tc_info.num_tc); 5293 dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu); 5294} 5295 5296static void hns3_set_cq_period_mode(struct hns3_nic_priv *priv, 5297 enum dim_cq_period_mode mode, bool is_tx) 5298{ 5299 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(priv->ae_handle); 5300 struct hnae3_handle *handle = priv->ae_handle; 5301 int i; 5302 5303 if (is_tx) { 5304 priv->tx_cqe_mode = mode; 5305 5306 for (i = 0; i < priv->vector_num; i++) 5307 priv->tqp_vector[i].tx_group.dim.mode = mode; 5308 } else { 5309 priv->rx_cqe_mode = mode; 5310 5311 for (i = 0; i < priv->vector_num; i++) 5312 priv->tqp_vector[i].rx_group.dim.mode = mode; 5313 } 5314 5315 if (hnae3_ae_dev_cq_supported(ae_dev)) { 5316 u32 new_mode; 5317 u64 reg; 5318 5319 new_mode = (mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE) ? 5320 HNS3_CQ_MODE_CQE : HNS3_CQ_MODE_EQE; 5321 reg = is_tx ? HNS3_GL1_CQ_MODE_REG : HNS3_GL0_CQ_MODE_REG; 5322 5323 writel(new_mode, handle->kinfo.io_base + reg); 5324 } 5325} 5326 5327void hns3_cq_period_mode_init(struct hns3_nic_priv *priv, 5328 enum dim_cq_period_mode tx_mode, 5329 enum dim_cq_period_mode rx_mode) 5330{ 5331 hns3_set_cq_period_mode(priv, tx_mode, true); 5332 hns3_set_cq_period_mode(priv, rx_mode, false); 5333} 5334 5335static void hns3_state_init(struct hnae3_handle *handle) 5336{ 5337 struct hnae3_ae_dev *ae_dev = hns3_get_ae_dev(handle); 5338 struct net_device *netdev = handle->kinfo.netdev; 5339 struct hns3_nic_priv *priv = netdev_priv(netdev); 5340 5341 set_bit(HNS3_NIC_STATE_INITED, &priv->state); 5342 5343 if (test_bit(HNAE3_DEV_SUPPORT_TX_PUSH_B, ae_dev->caps)) 5344 set_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state); 5345 5346 if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) 5347 set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->supported_pflags); 5348 5349 if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps)) 5350 set_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state); 5351 5352 if (hnae3_ae_dev_rxd_adv_layout_supported(ae_dev)) 5353 set_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state); 5354} 5355 5356static void hns3_state_uninit(struct hnae3_handle *handle) 5357{ 5358 struct hns3_nic_priv *priv = handle->priv; 5359 5360 clear_bit(HNS3_NIC_STATE_INITED, &priv->state); 5361} 5362 5363static int hns3_client_init(struct hnae3_handle *handle) 5364{ 5365 struct pci_dev *pdev = handle->pdev; 5366 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev); 5367 u16 alloc_tqps, max_rss_size; 5368 struct hns3_nic_priv *priv; 5369 struct net_device *netdev; 5370 int ret; 5371 5372 ae_dev->handle = handle; 5373 5374 handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps, 5375 &max_rss_size); 5376 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps); 5377 if (!netdev) 5378 return -ENOMEM; 5379 5380 priv = netdev_priv(netdev); 5381 priv->dev = &pdev->dev; 5382 priv->netdev = netdev; 5383 priv->ae_handle = handle; 5384 priv->tx_timeout_count = 0; 5385 priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num; 5386 priv->min_tx_copybreak = 0; 5387 priv->min_tx_spare_buf_size = 0; 5388 set_bit(HNS3_NIC_STATE_DOWN, &priv->state); 5389 5390 handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL); 5391 5392 handle->kinfo.netdev = netdev; 5393 handle->priv = (void *)priv; 5394 5395 hns3_init_mac_addr(netdev); 5396 5397 hns3_set_default_feature(netdev); 5398 5399 netdev->watchdog_timeo = HNS3_TX_TIMEOUT; 5400 netdev->priv_flags |= IFF_UNICAST_FLT; 5401 netdev->netdev_ops = &hns3_nic_netdev_ops; 5402 SET_NETDEV_DEV(netdev, &pdev->dev); 5403 hns3_ethtool_set_ops(netdev); 5404 5405 /* Carrier off reporting is important to ethtool even BEFORE open */ 5406 netif_carrier_off(netdev); 5407 5408 ret = hns3_get_ring_config(priv); 5409 if (ret) { 5410 ret = -ENOMEM; 5411 goto out_get_ring_cfg; 5412 } 5413 5414 hns3_nic_init_coal_cfg(priv); 5415 5416 ret = hns3_nic_alloc_vector_data(priv); 5417 if (ret) { 5418 ret = -ENOMEM; 5419 goto out_alloc_vector_data; 5420 } 5421 5422 ret = hns3_nic_init_vector_data(priv); 5423 if (ret) { 5424 ret = -ENOMEM; 5425 goto out_init_vector_data; 5426 } 5427 5428 ret = hns3_init_all_ring(priv); 5429 if (ret) { 5430 ret = -ENOMEM; 5431 goto out_init_ring; 5432 } 5433 5434 hns3_cq_period_mode_init(priv, DIM_CQ_PERIOD_MODE_START_FROM_EQE, 5435 DIM_CQ_PERIOD_MODE_START_FROM_EQE); 5436 5437 ret = hns3_init_phy(netdev); 5438 if (ret) 5439 goto out_init_phy; 5440 5441 /* the device can work without cpu rmap, only aRFS needs it */ 5442 ret = hns3_set_rx_cpu_rmap(netdev); 5443 if (ret) 5444 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret); 5445 5446 ret = hns3_nic_init_irq(priv); 5447 if (ret) { 5448 dev_err(priv->dev, "init irq failed! ret=%d\n", ret); 5449 hns3_free_rx_cpu_rmap(netdev); 5450 goto out_init_irq_fail; 5451 } 5452 5453 ret = hns3_client_start(handle); 5454 if (ret) { 5455 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret); 5456 goto out_client_start; 5457 } 5458 5459 hns3_dcbnl_setup(handle); 5460 5461 ret = hns3_dbg_init(handle); 5462 if (ret) { 5463 dev_err(priv->dev, "failed to init debugfs, ret = %d\n", 5464 ret); 5465 goto out_client_start; 5466 } 5467 5468 netdev->max_mtu = HNS3_MAX_MTU(ae_dev->dev_specs.max_frm_size); 5469 5470 hns3_state_init(handle); 5471 5472 ret = register_netdev(netdev); 5473 if (ret) { 5474 dev_err(priv->dev, "probe register netdev fail!\n"); 5475 goto out_reg_netdev_fail; 5476 } 5477 5478 if (netif_msg_drv(handle)) 5479 hns3_info_show(priv); 5480 5481 return ret; 5482 5483out_reg_netdev_fail: 5484 hns3_state_uninit(handle); 5485 hns3_dbg_uninit(handle); 5486 hns3_client_stop(handle); 5487out_client_start: 5488 hns3_free_rx_cpu_rmap(netdev); 5489 hns3_nic_uninit_irq(priv); 5490out_init_irq_fail: 5491 hns3_uninit_phy(netdev); 5492out_init_phy: 5493 hns3_uninit_all_ring(priv); 5494out_init_ring: 5495 hns3_nic_uninit_vector_data(priv); 5496out_init_vector_data: 5497 hns3_nic_dealloc_vector_data(priv); 5498out_alloc_vector_data: 5499 priv->ring = NULL; 5500out_get_ring_cfg: 5501 priv->ae_handle = NULL; 5502 free_netdev(netdev); 5503 return ret; 5504} 5505 5506static void hns3_client_uninit(struct hnae3_handle *handle, bool reset) 5507{ 5508 struct net_device *netdev = handle->kinfo.netdev; 5509 struct hns3_nic_priv *priv = netdev_priv(netdev); 5510 5511 if (netdev->reg_state != NETREG_UNINITIALIZED) 5512 unregister_netdev(netdev); 5513 5514 hns3_client_stop(handle); 5515 5516 hns3_uninit_phy(netdev); 5517 5518 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) { 5519 netdev_warn(netdev, "already uninitialized\n"); 5520 goto out_netdev_free; 5521 } 5522 5523 hns3_free_rx_cpu_rmap(netdev); 5524 5525 hns3_nic_uninit_irq(priv); 5526 5527 hns3_clear_all_ring(handle, true); 5528 5529 hns3_nic_uninit_vector_data(priv); 5530 5531 hns3_nic_dealloc_vector_data(priv); 5532 5533 hns3_uninit_all_ring(priv); 5534 5535 hns3_put_ring_config(priv); 5536 5537out_netdev_free: 5538 hns3_dbg_uninit(handle); 5539 free_netdev(netdev); 5540} 5541 5542static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup) 5543{ 5544 struct net_device *netdev = handle->kinfo.netdev; 5545 5546 if (!netdev) 5547 return; 5548 5549 if (linkup) { 5550 netif_tx_wake_all_queues(netdev); 5551 netif_carrier_on(netdev); 5552 if (netif_msg_link(handle)) 5553 netdev_info(netdev, "link up\n"); 5554 } else { 5555 netif_carrier_off(netdev); 5556 netif_tx_stop_all_queues(netdev); 5557 if (netif_msg_link(handle)) 5558 netdev_info(netdev, "link down\n"); 5559 } 5560} 5561 5562static void hns3_clear_tx_ring(struct hns3_enet_ring *ring) 5563{ 5564 while (ring->next_to_clean != ring->next_to_use) { 5565 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0; 5566 hns3_free_buffer_detach(ring, ring->next_to_clean, 0); 5567 ring_ptr_move_fw(ring, next_to_clean); 5568 } 5569 5570 ring->pending_buf = 0; 5571} 5572 5573static int hns3_clear_rx_ring(struct hns3_enet_ring *ring) 5574{ 5575 struct hns3_desc_cb res_cbs; 5576 int ret; 5577 5578 while (ring->next_to_use != ring->next_to_clean) { 5579 /* When a buffer is not reused, it's memory has been 5580 * freed in hns3_handle_rx_bd or will be freed by 5581 * stack, so we need to replace the buffer here. 5582 */ 5583 if (!ring->desc_cb[ring->next_to_use].reuse_flag) { 5584 ret = hns3_alloc_and_map_buffer(ring, &res_cbs); 5585 if (ret) { 5586 hns3_ring_stats_update(ring, sw_err_cnt); 5587 /* if alloc new buffer fail, exit directly 5588 * and reclear in up flow. 5589 */ 5590 netdev_warn(ring_to_netdev(ring), 5591 "reserve buffer map failed, ret = %d\n", 5592 ret); 5593 return ret; 5594 } 5595 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs); 5596 } 5597 ring_ptr_move_fw(ring, next_to_use); 5598 } 5599 5600 /* Free the pending skb in rx ring */ 5601 if (ring->skb) { 5602 dev_kfree_skb_any(ring->skb); 5603 ring->skb = NULL; 5604 ring->pending_buf = 0; 5605 } 5606 5607 return 0; 5608} 5609 5610static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring) 5611{ 5612 while (ring->next_to_use != ring->next_to_clean) { 5613 /* When a buffer is not reused, it's memory has been 5614 * freed in hns3_handle_rx_bd or will be freed by 5615 * stack, so only need to unmap the buffer here. 5616 */ 5617 if (!ring->desc_cb[ring->next_to_use].reuse_flag) { 5618 hns3_unmap_buffer(ring, 5619 &ring->desc_cb[ring->next_to_use]); 5620 ring->desc_cb[ring->next_to_use].dma = 0; 5621 } 5622 5623 ring_ptr_move_fw(ring, next_to_use); 5624 } 5625} 5626 5627static void hns3_clear_all_ring(struct hnae3_handle *h, bool force) 5628{ 5629 struct net_device *ndev = h->kinfo.netdev; 5630 struct hns3_nic_priv *priv = netdev_priv(ndev); 5631 u32 i; 5632 5633 for (i = 0; i < h->kinfo.num_tqps; i++) { 5634 struct hns3_enet_ring *ring; 5635 5636 ring = &priv->ring[i]; 5637 hns3_clear_tx_ring(ring); 5638 5639 ring = &priv->ring[i + h->kinfo.num_tqps]; 5640 /* Continue to clear other rings even if clearing some 5641 * rings failed. 5642 */ 5643 if (force) 5644 hns3_force_clear_rx_ring(ring); 5645 else 5646 hns3_clear_rx_ring(ring); 5647 } 5648} 5649 5650int hns3_nic_reset_all_ring(struct hnae3_handle *h) 5651{ 5652 struct net_device *ndev = h->kinfo.netdev; 5653 struct hns3_nic_priv *priv = netdev_priv(ndev); 5654 struct hns3_enet_ring *rx_ring; 5655 int i, j; 5656 int ret; 5657 5658 ret = h->ae_algo->ops->reset_queue(h); 5659 if (ret) 5660 return ret; 5661 5662 for (i = 0; i < h->kinfo.num_tqps; i++) { 5663 hns3_init_ring_hw(&priv->ring[i]); 5664 5665 /* We need to clear tx ring here because self test will 5666 * use the ring and will not run down before up 5667 */ 5668 hns3_clear_tx_ring(&priv->ring[i]); 5669 priv->ring[i].next_to_clean = 0; 5670 priv->ring[i].next_to_use = 0; 5671 priv->ring[i].last_to_use = 0; 5672 5673 rx_ring = &priv->ring[i + h->kinfo.num_tqps]; 5674 hns3_init_ring_hw(rx_ring); 5675 ret = hns3_clear_rx_ring(rx_ring); 5676 if (ret) 5677 return ret; 5678 5679 /* We can not know the hardware head and tail when this 5680 * function is called in reset flow, so we reuse all desc. 5681 */ 5682 for (j = 0; j < rx_ring->desc_num; j++) 5683 hns3_reuse_buffer(rx_ring, j); 5684 5685 rx_ring->next_to_clean = 0; 5686 rx_ring->next_to_use = 0; 5687 } 5688 5689 hns3_init_tx_ring_tc(priv); 5690 5691 return 0; 5692} 5693 5694static int hns3_reset_notify_down_enet(struct hnae3_handle *handle) 5695{ 5696 struct hnae3_knic_private_info *kinfo = &handle->kinfo; 5697 struct net_device *ndev = kinfo->netdev; 5698 struct hns3_nic_priv *priv = netdev_priv(ndev); 5699 5700 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state)) 5701 return 0; 5702 5703 if (!netif_running(ndev)) 5704 return 0; 5705 5706 return hns3_nic_net_stop(ndev); 5707} 5708 5709static int hns3_reset_notify_up_enet(struct hnae3_handle *handle) 5710{ 5711 struct hnae3_knic_private_info *kinfo = &handle->kinfo; 5712 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev); 5713 int ret = 0; 5714 5715 if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) { 5716 netdev_err(kinfo->netdev, "device is not initialized yet\n"); 5717 return -EFAULT; 5718 } 5719 5720 clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state); 5721 5722 if (netif_running(kinfo->netdev)) { 5723 ret = hns3_nic_net_open(kinfo->netdev); 5724 if (ret) { 5725 set_bit(HNS3_NIC_STATE_RESETTING, &priv->state); 5726 netdev_err(kinfo->netdev, 5727 "net up fail, ret=%d!\n", ret); 5728 return ret; 5729 } 5730 } 5731 5732 return ret; 5733} 5734 5735static int hns3_reset_notify_init_enet(struct hnae3_handle *handle) 5736{ 5737 struct net_device *netdev = handle->kinfo.netdev; 5738 struct hns3_nic_priv *priv = netdev_priv(netdev); 5739 int ret; 5740 5741 /* Carrier off reporting is important to ethtool even BEFORE open */ 5742 netif_carrier_off(netdev); 5743 5744 ret = hns3_get_ring_config(priv); 5745 if (ret) 5746 return ret; 5747 5748 ret = hns3_nic_alloc_vector_data(priv); 5749 if (ret) 5750 goto err_put_ring; 5751 5752 ret = hns3_nic_init_vector_data(priv); 5753 if (ret) 5754 goto err_dealloc_vector; 5755 5756 ret = hns3_init_all_ring(priv); 5757 if (ret) 5758 goto err_uninit_vector; 5759 5760 hns3_cq_period_mode_init(priv, priv->tx_cqe_mode, priv->rx_cqe_mode); 5761 5762 /* the device can work without cpu rmap, only aRFS needs it */ 5763 ret = hns3_set_rx_cpu_rmap(netdev); 5764 if (ret) 5765 dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret); 5766 5767 ret = hns3_nic_init_irq(priv); 5768 if (ret) { 5769 dev_err(priv->dev, "init irq failed! ret=%d\n", ret); 5770 hns3_free_rx_cpu_rmap(netdev); 5771 goto err_init_irq_fail; 5772 } 5773 5774 if (!hns3_is_phys_func(handle->pdev)) 5775 hns3_init_mac_addr(netdev); 5776 5777 ret = hns3_client_start(handle); 5778 if (ret) { 5779 dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret); 5780 goto err_client_start_fail; 5781 } 5782 5783 set_bit(HNS3_NIC_STATE_INITED, &priv->state); 5784 5785 return ret; 5786 5787err_client_start_fail: 5788 hns3_free_rx_cpu_rmap(netdev); 5789 hns3_nic_uninit_irq(priv); 5790err_init_irq_fail: 5791 hns3_uninit_all_ring(priv); 5792err_uninit_vector: 5793 hns3_nic_uninit_vector_data(priv); 5794err_dealloc_vector: 5795 hns3_nic_dealloc_vector_data(priv); 5796err_put_ring: 5797 hns3_put_ring_config(priv); 5798 5799 return ret; 5800} 5801 5802static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) 5803{ 5804 struct net_device *netdev = handle->kinfo.netdev; 5805 struct hns3_nic_priv *priv = netdev_priv(netdev); 5806 5807 if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 5808 hns3_nic_net_stop(netdev); 5809 5810 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) { 5811 netdev_warn(netdev, "already uninitialized\n"); 5812 return 0; 5813 } 5814 5815 hns3_free_rx_cpu_rmap(netdev); 5816 hns3_nic_uninit_irq(priv); 5817 hns3_clear_all_ring(handle, true); 5818 hns3_reset_tx_queue(priv->ae_handle); 5819 5820 hns3_nic_uninit_vector_data(priv); 5821 5822 hns3_nic_dealloc_vector_data(priv); 5823 5824 hns3_uninit_all_ring(priv); 5825 5826 hns3_put_ring_config(priv); 5827 5828 return 0; 5829} 5830 5831int hns3_reset_notify(struct hnae3_handle *handle, 5832 enum hnae3_reset_notify_type type) 5833{ 5834 int ret = 0; 5835 5836 switch (type) { 5837 case HNAE3_UP_CLIENT: 5838 ret = hns3_reset_notify_up_enet(handle); 5839 break; 5840 case HNAE3_DOWN_CLIENT: 5841 ret = hns3_reset_notify_down_enet(handle); 5842 break; 5843 case HNAE3_INIT_CLIENT: 5844 ret = hns3_reset_notify_init_enet(handle); 5845 break; 5846 case HNAE3_UNINIT_CLIENT: 5847 ret = hns3_reset_notify_uninit_enet(handle); 5848 break; 5849 default: 5850 break; 5851 } 5852 5853 return ret; 5854} 5855 5856static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num, 5857 bool rxfh_configured) 5858{ 5859 int ret; 5860 5861 ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num, 5862 rxfh_configured); 5863 if (ret) { 5864 dev_err(&handle->pdev->dev, 5865 "Change tqp num(%u) fail.\n", new_tqp_num); 5866 return ret; 5867 } 5868 5869 ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT); 5870 if (ret) 5871 return ret; 5872 5873 ret = hns3_reset_notify(handle, HNAE3_UP_CLIENT); 5874 if (ret) 5875 hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT); 5876 5877 return ret; 5878} 5879 5880int hns3_set_channels(struct net_device *netdev, 5881 struct ethtool_channels *ch) 5882{ 5883 struct hnae3_handle *h = hns3_get_handle(netdev); 5884 struct hnae3_knic_private_info *kinfo = &h->kinfo; 5885 bool rxfh_configured = netif_is_rxfh_configured(netdev); 5886 u32 new_tqp_num = ch->combined_count; 5887 u16 org_tqp_num; 5888 int ret; 5889 5890 if (hns3_nic_resetting(netdev)) 5891 return -EBUSY; 5892 5893 if (ch->rx_count || ch->tx_count) 5894 return -EINVAL; 5895 5896 if (kinfo->tc_info.mqprio_active) { 5897 dev_err(&netdev->dev, 5898 "it's not allowed to set channels via ethtool when MQPRIO mode is on\n"); 5899 return -EINVAL; 5900 } 5901 5902 if (new_tqp_num > hns3_get_max_available_channels(h) || 5903 new_tqp_num < 1) { 5904 dev_err(&netdev->dev, 5905 "Change tqps fail, the tqp range is from 1 to %u", 5906 hns3_get_max_available_channels(h)); 5907 return -EINVAL; 5908 } 5909 5910 if (kinfo->rss_size == new_tqp_num) 5911 return 0; 5912 5913 netif_dbg(h, drv, netdev, 5914 "set channels: tqp_num=%u, rxfh=%d\n", 5915 new_tqp_num, rxfh_configured); 5916 5917 ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT); 5918 if (ret) 5919 return ret; 5920 5921 ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT); 5922 if (ret) 5923 return ret; 5924 5925 org_tqp_num = h->kinfo.num_tqps; 5926 ret = hns3_change_channels(h, new_tqp_num, rxfh_configured); 5927 if (ret) { 5928 int ret1; 5929 5930 netdev_warn(netdev, 5931 "Change channels fail, revert to old value\n"); 5932 ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured); 5933 if (ret1) { 5934 netdev_err(netdev, 5935 "revert to old channel fail\n"); 5936 return ret1; 5937 } 5938 5939 return ret; 5940 } 5941 5942 return 0; 5943} 5944 5945void hns3_external_lb_prepare(struct net_device *ndev, bool if_running) 5946{ 5947 struct hns3_nic_priv *priv = netdev_priv(ndev); 5948 5949 if (!if_running) 5950 return; 5951 5952 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 5953 return; 5954 5955 netif_carrier_off(ndev); 5956 netif_tx_disable(ndev); 5957 5958 hns3_disable_irqs_and_tqps(ndev); 5959 5960 /* delay ring buffer clearing to hns3_reset_notify_uninit_enet 5961 * during reset process, because driver may not be able 5962 * to disable the ring through firmware when downing the netdev. 5963 */ 5964 if (!hns3_nic_resetting(ndev)) 5965 hns3_nic_reset_all_ring(priv->ae_handle); 5966 5967 hns3_reset_tx_queue(priv->ae_handle); 5968} 5969 5970void hns3_external_lb_restore(struct net_device *ndev, bool if_running) 5971{ 5972 struct hns3_nic_priv *priv = netdev_priv(ndev); 5973 struct hnae3_handle *h = priv->ae_handle; 5974 5975 if (!if_running) 5976 return; 5977 5978 if (hns3_nic_resetting(ndev)) 5979 return; 5980 5981 if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) 5982 return; 5983 5984 if (hns3_nic_reset_all_ring(priv->ae_handle)) 5985 return; 5986 5987 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state); 5988 5989 hns3_enable_irqs_and_tqps(ndev); 5990 5991 netif_tx_wake_all_queues(ndev); 5992 5993 if (h->ae_algo->ops->get_status(h)) 5994 netif_carrier_on(ndev); 5995} 5996 5997static const struct hns3_hw_error_info hns3_hw_err[] = { 5998 { .type = HNAE3_PPU_POISON_ERROR, 5999 .msg = "PPU poison" }, 6000 { .type = HNAE3_CMDQ_ECC_ERROR, 6001 .msg = "IMP CMDQ error" }, 6002 { .type = HNAE3_IMP_RD_POISON_ERROR, 6003 .msg = "IMP RD poison" }, 6004 { .type = HNAE3_ROCEE_AXI_RESP_ERROR, 6005 .msg = "ROCEE AXI RESP error" }, 6006}; 6007 6008static void hns3_process_hw_error(struct hnae3_handle *handle, 6009 enum hnae3_hw_error_type type) 6010{ 6011 u32 i; 6012 6013 for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) { 6014 if (hns3_hw_err[i].type == type) { 6015 dev_err(&handle->pdev->dev, "Detected %s!\n", 6016 hns3_hw_err[i].msg); 6017 break; 6018 } 6019 } 6020} 6021 6022static const struct hnae3_client_ops client_ops = { 6023 .init_instance = hns3_client_init, 6024 .uninit_instance = hns3_client_uninit, 6025 .link_status_change = hns3_link_status_change, 6026 .reset_notify = hns3_reset_notify, 6027 .process_hw_error = hns3_process_hw_error, 6028}; 6029 6030/* hns3_init_module - Driver registration routine 6031 * hns3_init_module is the first routine called when the driver is 6032 * loaded. All it does is register with the PCI subsystem. 6033 */ 6034static int __init hns3_init_module(void) 6035{ 6036 int ret; 6037 6038 pr_debug("%s: %s - version\n", hns3_driver_name, hns3_driver_string); 6039 pr_debug("%s: %s\n", hns3_driver_name, hns3_copyright); 6040 6041 client.type = HNAE3_CLIENT_KNIC; 6042 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s", 6043 hns3_driver_name); 6044 6045 client.ops = &client_ops; 6046 6047 INIT_LIST_HEAD(&client.node); 6048 6049 hns3_dbg_register_debugfs(hns3_driver_name); 6050 6051 ret = hnae3_register_client(&client); 6052 if (ret) 6053 goto err_reg_client; 6054 6055 ret = pci_register_driver(&hns3_driver); 6056 if (ret) 6057 goto err_reg_driver; 6058 6059 return ret; 6060 6061err_reg_driver: 6062 hnae3_unregister_client(&client); 6063err_reg_client: 6064 hns3_dbg_unregister_debugfs(); 6065 return ret; 6066} 6067module_init(hns3_init_module); 6068 6069/* hns3_exit_module - Driver exit cleanup routine 6070 * hns3_exit_module is called just before the driver is removed 6071 * from memory. 6072 */ 6073static void __exit hns3_exit_module(void) 6074{ 6075 hnae3_acquire_unload_lock(); 6076 pci_unregister_driver(&hns3_driver); 6077 hnae3_unregister_client(&client); 6078 hns3_dbg_unregister_debugfs(); 6079 hnae3_release_unload_lock(); 6080} 6081module_exit(hns3_exit_module); 6082 6083MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver"); 6084MODULE_AUTHOR("Huawei Tech. Co., Ltd."); 6085MODULE_LICENSE("GPL"); 6086MODULE_ALIAS("pci:hns-nic");
~/
Read-only file preview.