FILE
virtio_net.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/virtio_net.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* A network driver using virtio. 3 * 4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5 */ 6//#define DEBUG 7#include <linux/netdevice.h> 8#include <linux/etherdevice.h> 9#include <linux/ethtool.h> 10#include <linux/module.h> 11#include <linux/virtio.h> 12#include <linux/virtio_net.h> 13#include <linux/bpf.h> 14#include <linux/bpf_trace.h> 15#include <linux/scatterlist.h> 16#include <linux/if_vlan.h> 17#include <linux/slab.h> 18#include <linux/cpu.h> 19#include <linux/average.h> 20#include <linux/filter.h> 21#include <linux/kernel.h> 22#include <linux/dim.h> 23#include <net/route.h> 24#include <net/xdp.h> 25#include <net/net_failover.h> 26#include <net/netdev_rx_queue.h> 27#include <net/netdev_queues.h> 28#include <net/xdp_sock_drv.h> 29 30static int napi_weight = NAPI_POLL_WEIGHT; 31module_param(napi_weight, int, 0444); 32 33static bool csum = true, gso = true, napi_tx = true; 34module_param(csum, bool, 0444); 35module_param(gso, bool, 0444); 36module_param(napi_tx, bool, 0644); 37 38#define VIRTIO_OFFLOAD_MAP_MIN 46 39#define VIRTIO_OFFLOAD_MAP_MAX 47 40#define VIRTIO_FEATURES_MAP_MIN 65 41#define VIRTIO_O2F_DELTA (VIRTIO_FEATURES_MAP_MIN - \ 42 VIRTIO_OFFLOAD_MAP_MIN) 43 44static bool virtio_is_mapped_offload(unsigned int obit) 45{ 46 return obit >= VIRTIO_OFFLOAD_MAP_MIN && 47 obit <= VIRTIO_OFFLOAD_MAP_MAX; 48} 49 50static unsigned int virtio_offload_to_feature(unsigned int obit) 51{ 52 return virtio_is_mapped_offload(obit) ? obit + VIRTIO_O2F_DELTA : obit; 53} 54 55/* FIXME: MTU in config. */ 56#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 57#define GOOD_COPY_LEN 128 58 59#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 60 61/* Separating two types of XDP xmit */ 62#define VIRTIO_XDP_TX BIT(0) 63#define VIRTIO_XDP_REDIR BIT(1) 64 65/* RX packet size EWMA. The average packet size is used to determine the packet 66 * buffer size when refilling RX rings. As the entire RX ring may be refilled 67 * at once, the weight is chosen so that the EWMA will be insensitive to short- 68 * term, transient changes in packet size. 69 */ 70DECLARE_EWMA(pkt_len, 0, 64) 71 72#define VIRTNET_DRIVER_VERSION "1.0.0" 73 74static const unsigned long guest_offloads[] = { 75 VIRTIO_NET_F_GUEST_TSO4, 76 VIRTIO_NET_F_GUEST_TSO6, 77 VIRTIO_NET_F_GUEST_ECN, 78 VIRTIO_NET_F_GUEST_UFO, 79 VIRTIO_NET_F_GUEST_CSUM, 80 VIRTIO_NET_F_GUEST_USO4, 81 VIRTIO_NET_F_GUEST_USO6, 82 VIRTIO_NET_F_GUEST_HDRLEN, 83 VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED, 84 VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED, 85}; 86 87#define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 88 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 89 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 90 (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ 91 (1ULL << VIRTIO_NET_F_GUEST_USO4) | \ 92 (1ULL << VIRTIO_NET_F_GUEST_USO6) | \ 93 (1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_MAPPED) | \ 94 (1ULL << VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM_MAPPED)) 95 96struct virtnet_stat_desc { 97 char desc[ETH_GSTRING_LEN]; 98 size_t offset; 99 size_t qstat_offset; 100}; 101 102struct virtnet_sq_free_stats { 103 u64 packets; 104 u64 bytes; 105 u64 napi_packets; 106 u64 napi_bytes; 107 u64 xsk; 108}; 109 110struct virtnet_sq_stats { 111 struct u64_stats_sync syncp; 112 u64_stats_t packets; 113 u64_stats_t bytes; 114 u64_stats_t xdp_tx; 115 u64_stats_t xdp_tx_drops; 116 u64_stats_t kicks; 117 u64_stats_t tx_timeouts; 118 u64_stats_t stop; 119 u64_stats_t wake; 120}; 121 122struct virtnet_rq_stats { 123 struct u64_stats_sync syncp; 124 u64_stats_t packets; 125 u64_stats_t bytes; 126 u64_stats_t drops; 127 u64_stats_t xdp_packets; 128 u64_stats_t xdp_tx; 129 u64_stats_t xdp_redirects; 130 u64_stats_t xdp_drops; 131 u64_stats_t kicks; 132}; 133 134#define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1} 135#define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1} 136 137#define VIRTNET_SQ_STAT_QSTAT(name, m) \ 138 { \ 139 name, \ 140 offsetof(struct virtnet_sq_stats, m), \ 141 offsetof(struct netdev_queue_stats_tx, m), \ 142 } 143 144#define VIRTNET_RQ_STAT_QSTAT(name, m) \ 145 { \ 146 name, \ 147 offsetof(struct virtnet_rq_stats, m), \ 148 offsetof(struct netdev_queue_stats_rx, m), \ 149 } 150 151static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 152 VIRTNET_SQ_STAT("xdp_tx", xdp_tx), 153 VIRTNET_SQ_STAT("xdp_tx_drops", xdp_tx_drops), 154 VIRTNET_SQ_STAT("kicks", kicks), 155 VIRTNET_SQ_STAT("tx_timeouts", tx_timeouts), 156}; 157 158static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 159 VIRTNET_RQ_STAT("drops", drops), 160 VIRTNET_RQ_STAT("xdp_packets", xdp_packets), 161 VIRTNET_RQ_STAT("xdp_tx", xdp_tx), 162 VIRTNET_RQ_STAT("xdp_redirects", xdp_redirects), 163 VIRTNET_RQ_STAT("xdp_drops", xdp_drops), 164 VIRTNET_RQ_STAT("kicks", kicks), 165}; 166 167static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = { 168 VIRTNET_SQ_STAT_QSTAT("packets", packets), 169 VIRTNET_SQ_STAT_QSTAT("bytes", bytes), 170 VIRTNET_SQ_STAT_QSTAT("stop", stop), 171 VIRTNET_SQ_STAT_QSTAT("wake", wake), 172}; 173 174static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = { 175 VIRTNET_RQ_STAT_QSTAT("packets", packets), 176 VIRTNET_RQ_STAT_QSTAT("bytes", bytes), 177}; 178 179#define VIRTNET_STATS_DESC_CQ(name) \ 180 {#name, offsetof(struct virtio_net_stats_cvq, name), -1} 181 182#define VIRTNET_STATS_DESC_RX(class, name) \ 183 {#name, offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), -1} 184 185#define VIRTNET_STATS_DESC_TX(class, name) \ 186 {#name, offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), -1} 187 188 189static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = { 190 VIRTNET_STATS_DESC_CQ(command_num), 191 VIRTNET_STATS_DESC_CQ(ok_num), 192}; 193 194static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = { 195 VIRTNET_STATS_DESC_RX(basic, packets), 196 VIRTNET_STATS_DESC_RX(basic, bytes), 197 198 VIRTNET_STATS_DESC_RX(basic, notifications), 199 VIRTNET_STATS_DESC_RX(basic, interrupts), 200}; 201 202static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = { 203 VIRTNET_STATS_DESC_TX(basic, packets), 204 VIRTNET_STATS_DESC_TX(basic, bytes), 205 206 VIRTNET_STATS_DESC_TX(basic, notifications), 207 VIRTNET_STATS_DESC_TX(basic, interrupts), 208}; 209 210static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = { 211 VIRTNET_STATS_DESC_RX(csum, needs_csum), 212}; 213 214static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = { 215 VIRTNET_STATS_DESC_TX(gso, gso_packets_noseg), 216 VIRTNET_STATS_DESC_TX(gso, gso_bytes_noseg), 217}; 218 219static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = { 220 VIRTNET_STATS_DESC_RX(speed, ratelimit_bytes), 221}; 222 223static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = { 224 VIRTNET_STATS_DESC_TX(speed, ratelimit_bytes), 225}; 226 227#define VIRTNET_STATS_DESC_RX_QSTAT(class, name, qstat_field) \ 228 { \ 229 #name, \ 230 offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), \ 231 offsetof(struct netdev_queue_stats_rx, qstat_field), \ 232 } 233 234#define VIRTNET_STATS_DESC_TX_QSTAT(class, name, qstat_field) \ 235 { \ 236 #name, \ 237 offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), \ 238 offsetof(struct netdev_queue_stats_tx, qstat_field), \ 239 } 240 241static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = { 242 VIRTNET_STATS_DESC_RX_QSTAT(basic, drops, hw_drops), 243 VIRTNET_STATS_DESC_RX_QSTAT(basic, drop_overruns, hw_drop_overruns), 244}; 245 246static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = { 247 VIRTNET_STATS_DESC_TX_QSTAT(basic, drops, hw_drops), 248 VIRTNET_STATS_DESC_TX_QSTAT(basic, drop_malformed, hw_drop_errors), 249}; 250 251static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = { 252 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_valid, csum_unnecessary), 253 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_none, csum_none), 254 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_bad, csum_bad), 255}; 256 257static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = { 258 VIRTNET_STATS_DESC_TX_QSTAT(csum, csum_none, csum_none), 259 VIRTNET_STATS_DESC_TX_QSTAT(csum, needs_csum, needs_csum), 260}; 261 262static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = { 263 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets, hw_gro_packets), 264 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes, hw_gro_bytes), 265 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets_coalesced, hw_gro_wire_packets), 266 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes_coalesced, hw_gro_wire_bytes), 267}; 268 269static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = { 270 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_packets, hw_gso_packets), 271 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_bytes, hw_gso_bytes), 272 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments, hw_gso_wire_packets), 273 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments_bytes, hw_gso_wire_bytes), 274}; 275 276static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = { 277 VIRTNET_STATS_DESC_RX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits), 278}; 279 280static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = { 281 VIRTNET_STATS_DESC_TX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits), 282}; 283 284#define VIRTNET_Q_TYPE_RX 0 285#define VIRTNET_Q_TYPE_TX 1 286#define VIRTNET_Q_TYPE_CQ 2 287 288struct virtnet_interrupt_coalesce { 289 u32 max_packets; 290 u32 max_usecs; 291}; 292 293/* The dma information of pages allocated at a time. */ 294struct virtnet_rq_dma { 295 dma_addr_t addr; 296 u32 ref; 297 u16 len; 298 u16 need_sync; 299}; 300 301/* Internal representation of a send virtqueue */ 302struct send_queue { 303 /* Virtqueue associated with this send _queue */ 304 struct virtqueue *vq; 305 306 /* TX: fragments + linear part + virtio header */ 307 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 308 309 /* Name of the send queue: output.$index */ 310 char name[16]; 311 312 struct virtnet_sq_stats stats; 313 314 struct virtnet_interrupt_coalesce intr_coal; 315 316 struct napi_struct napi; 317 318 /* Record whether sq is in reset state. */ 319 bool reset; 320 321 struct xsk_buff_pool *xsk_pool; 322 323 dma_addr_t xsk_hdr_dma_addr; 324}; 325 326/* Internal representation of a receive virtqueue */ 327struct receive_queue { 328 /* Virtqueue associated with this receive_queue */ 329 struct virtqueue *vq; 330 331 struct napi_struct napi; 332 333 struct bpf_prog __rcu *xdp_prog; 334 335 struct virtnet_rq_stats stats; 336 337 /* The number of rx notifications */ 338 u16 calls; 339 340 /* Is dynamic interrupt moderation enabled? */ 341 bool dim_enabled; 342 343 /* Used to protect dim_enabled and inter_coal */ 344 struct mutex dim_lock; 345 346 /* Dynamic Interrupt Moderation */ 347 struct dim dim; 348 349 u32 packets_in_napi; 350 351 struct virtnet_interrupt_coalesce intr_coal; 352 353 /* Chain pages by the private ptr. */ 354 struct page *pages; 355 356 /* Average packet length for mergeable receive buffers. */ 357 struct ewma_pkt_len mrg_avg_pkt_len; 358 359 /* Page frag for packet buffer allocation. */ 360 struct page_frag alloc_frag; 361 362 /* RX: fragments + linear part + virtio header */ 363 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 364 365 /* Min single buffer size for mergeable buffers case. */ 366 unsigned int min_buf_len; 367 368 /* Name of this receive queue: input.$index */ 369 char name[16]; 370 371 struct xdp_rxq_info xdp_rxq; 372 373 /* Record the last dma info to free after new pages is allocated. */ 374 struct virtnet_rq_dma *last_dma; 375 376 struct xsk_buff_pool *xsk_pool; 377 378 /* xdp rxq used by xsk */ 379 struct xdp_rxq_info xsk_rxq_info; 380 381 struct xdp_buff **xsk_buffs; 382}; 383 384/* Control VQ buffers: protected by the rtnl lock */ 385struct control_buf { 386 struct virtio_net_ctrl_hdr hdr; 387 virtio_net_ctrl_ack status; 388}; 389 390struct virtnet_info { 391 struct virtio_device *vdev; 392 struct virtqueue *cvq; 393 struct net_device *dev; 394 struct send_queue *sq; 395 struct receive_queue *rq; 396 unsigned int status; 397 398 /* Max # of queue pairs supported by the device */ 399 u16 max_queue_pairs; 400 401 /* # of queue pairs currently used by the driver */ 402 u16 curr_queue_pairs; 403 404 /* # of XDP queue pairs currently used by the driver */ 405 u16 xdp_queue_pairs; 406 407 /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 408 bool xdp_enabled; 409 410 /* I like... big packets and I cannot lie! */ 411 bool big_packets; 412 413 /* number of sg entries allocated for big packets */ 414 unsigned int big_packets_num_skbfrags; 415 416 /* Host will merge rx buffers for big packets (shake it! shake it!) */ 417 bool mergeable_rx_bufs; 418 419 /* Host supports rss and/or hash report */ 420 bool has_rss; 421 bool has_rss_hash_report; 422 u8 rss_key_size; 423 u16 rss_indir_table_size; 424 u32 rss_hash_types_supported; 425 u32 rss_hash_types_saved; 426 427 /* Has control virtqueue */ 428 bool has_cvq; 429 430 /* Lock to protect the control VQ */ 431 struct mutex cvq_lock; 432 433 /* Host can handle any s/g split between our header and packet data */ 434 bool any_header_sg; 435 436 /* Packet virtio header size */ 437 u8 hdr_len; 438 439 /* UDP tunnel support */ 440 bool tx_tnl; 441 442 bool rx_tnl; 443 444 bool rx_tnl_csum; 445 446 /* Work struct for config space updates */ 447 struct work_struct config_work; 448 449 /* Work struct for setting rx mode */ 450 struct work_struct rx_mode_work; 451 452 /* OK to queue work setting RX mode? */ 453 bool rx_mode_work_enabled; 454 455 /* Does the affinity hint is set for virtqueues? */ 456 bool affinity_hint_set; 457 458 /* CPU hotplug instances for online & dead */ 459 struct hlist_node node; 460 struct hlist_node node_dead; 461 462 struct control_buf *ctrl; 463 464 /* Ethtool settings */ 465 u8 duplex; 466 u32 speed; 467 468 /* Is rx dynamic interrupt moderation enabled? */ 469 bool rx_dim_enabled; 470 471 /* Interrupt coalescing settings */ 472 struct virtnet_interrupt_coalesce intr_coal_tx; 473 struct virtnet_interrupt_coalesce intr_coal_rx; 474 475 unsigned long guest_offloads; 476 unsigned long guest_offloads_capable; 477 478 /* failover when STANDBY feature enabled */ 479 struct failover *failover; 480 481 u64 device_stats_cap; 482 483 struct virtio_net_rss_config_hdr *rss_hdr; 484 485 /* Must be last as it ends in a flexible-array member. */ 486 TRAILING_OVERLAP(struct virtio_net_rss_config_trailer, rss_trailer, hash_key_data, 487 u8 rss_hash_key_data[NETDEV_RSS_KEY_LEN]; 488 ); 489}; 490static_assert(offsetof(struct virtnet_info, rss_trailer.hash_key_data) == 491 offsetof(struct virtnet_info, rss_hash_key_data)); 492 493struct padded_vnet_hdr { 494 struct virtio_net_hdr_v1_hash hdr; 495 /* 496 * hdr is in a separate sg buffer, and data sg buffer shares same page 497 * with this header sg. This padding makes next sg 16 byte aligned 498 * after the header. 499 */ 500 char padding[12]; 501}; 502 503struct virtio_net_common_hdr { 504 union { 505 struct virtio_net_hdr hdr; 506 struct virtio_net_hdr_mrg_rxbuf mrg_hdr; 507 struct virtio_net_hdr_v1_hash hash_v1_hdr; 508 struct virtio_net_hdr_v1_hash_tunnel tnl_hdr; 509 }; 510}; 511 512static struct virtio_net_common_hdr xsk_hdr; 513 514static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf); 515static void virtnet_sq_free_unused_buf_done(struct virtqueue *vq); 516static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, 517 struct net_device *dev, 518 unsigned int *xdp_xmit, 519 struct virtnet_rq_stats *stats); 520static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq, 521 struct sk_buff *skb, u8 flags); 522static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb, 523 struct sk_buff *curr_skb, 524 struct page *page, void *buf, 525 int len, int truesize); 526static void virtnet_xsk_completed(struct send_queue *sq, int num); 527 528enum virtnet_xmit_type { 529 VIRTNET_XMIT_TYPE_SKB, 530 VIRTNET_XMIT_TYPE_SKB_ORPHAN, 531 VIRTNET_XMIT_TYPE_XDP, 532 VIRTNET_XMIT_TYPE_XSK, 533}; 534 535static size_t virtnet_rss_hdr_size(const struct virtnet_info *vi) 536{ 537 u16 indir_table_size = vi->has_rss ? vi->rss_indir_table_size : 1; 538 539 return struct_size(vi->rss_hdr, indirection_table, indir_table_size); 540} 541 542static size_t virtnet_rss_trailer_size(const struct virtnet_info *vi) 543{ 544 return struct_size(&vi->rss_trailer, hash_key_data, vi->rss_key_size); 545} 546 547/* We use the last two bits of the pointer to distinguish the xmit type. */ 548#define VIRTNET_XMIT_TYPE_MASK (BIT(0) | BIT(1)) 549 550#define VIRTIO_XSK_FLAG_OFFSET 2 551 552static enum virtnet_xmit_type virtnet_xmit_ptr_unpack(void **ptr) 553{ 554 unsigned long p = (unsigned long)*ptr; 555 556 *ptr = (void *)(p & ~VIRTNET_XMIT_TYPE_MASK); 557 558 return p & VIRTNET_XMIT_TYPE_MASK; 559} 560 561static void *virtnet_xmit_ptr_pack(void *ptr, enum virtnet_xmit_type type) 562{ 563 return (void *)((unsigned long)ptr | type); 564} 565 566static int virtnet_add_outbuf(struct send_queue *sq, int num, void *data, 567 enum virtnet_xmit_type type) 568{ 569 return virtqueue_add_outbuf(sq->vq, sq->sg, num, 570 virtnet_xmit_ptr_pack(data, type), 571 GFP_ATOMIC); 572} 573 574static u32 virtnet_ptr_to_xsk_buff_len(void *ptr) 575{ 576 return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET; 577} 578 579static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len) 580{ 581 sg_dma_address(sg) = addr; 582 sg_dma_len(sg) = len; 583} 584 585static void __free_old_xmit(struct send_queue *sq, struct netdev_queue *txq, 586 bool in_napi, struct virtnet_sq_free_stats *stats) 587{ 588 struct xdp_frame *frame; 589 struct sk_buff *skb; 590 unsigned int len; 591 void *ptr; 592 593 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 594 switch (virtnet_xmit_ptr_unpack(&ptr)) { 595 case VIRTNET_XMIT_TYPE_SKB: 596 skb = ptr; 597 598 pr_debug("Sent skb %p\n", skb); 599 stats->napi_packets++; 600 stats->napi_bytes += skb->len; 601 napi_consume_skb(skb, in_napi); 602 break; 603 604 case VIRTNET_XMIT_TYPE_SKB_ORPHAN: 605 skb = ptr; 606 607 stats->packets++; 608 stats->bytes += skb->len; 609 napi_consume_skb(skb, in_napi); 610 break; 611 612 case VIRTNET_XMIT_TYPE_XDP: 613 frame = ptr; 614 615 stats->packets++; 616 stats->bytes += xdp_get_frame_len(frame); 617 xdp_return_frame(frame); 618 break; 619 620 case VIRTNET_XMIT_TYPE_XSK: 621 stats->bytes += virtnet_ptr_to_xsk_buff_len(ptr); 622 stats->xsk++; 623 break; 624 } 625 } 626 netdev_tx_completed_queue(txq, stats->napi_packets, stats->napi_bytes); 627} 628 629static void virtnet_free_old_xmit(struct send_queue *sq, 630 struct netdev_queue *txq, 631 bool in_napi, 632 struct virtnet_sq_free_stats *stats) 633{ 634 __free_old_xmit(sq, txq, in_napi, stats); 635 636 if (stats->xsk) 637 virtnet_xsk_completed(sq, stats->xsk); 638} 639 640/* Converting between virtqueue no. and kernel tx/rx queue no. 641 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 642 */ 643static int vq2txq(struct virtqueue *vq) 644{ 645 return (vq->index - 1) / 2; 646} 647 648static int txq2vq(int txq) 649{ 650 return txq * 2 + 1; 651} 652 653static int vq2rxq(struct virtqueue *vq) 654{ 655 return vq->index / 2; 656} 657 658static int rxq2vq(int rxq) 659{ 660 return rxq * 2; 661} 662 663static int vq_type(struct virtnet_info *vi, int qid) 664{ 665 if (qid == vi->max_queue_pairs * 2) 666 return VIRTNET_Q_TYPE_CQ; 667 668 if (qid % 2) 669 return VIRTNET_Q_TYPE_TX; 670 671 return VIRTNET_Q_TYPE_RX; 672} 673 674static inline struct virtio_net_common_hdr * 675skb_vnet_common_hdr(struct sk_buff *skb) 676{ 677 return (struct virtio_net_common_hdr *)skb->cb; 678} 679 680/* 681 * private is used to chain pages for big packets, put the whole 682 * most recent used list in the beginning for reuse 683 */ 684static void give_pages(struct receive_queue *rq, struct page *page) 685{ 686 struct page *end; 687 688 /* Find end of list, sew whole thing into vi->rq.pages. */ 689 for (end = page; end->private; end = (struct page *)end->private); 690 end->private = (unsigned long)rq->pages; 691 rq->pages = page; 692} 693 694static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 695{ 696 struct page *p = rq->pages; 697 698 if (p) { 699 rq->pages = (struct page *)p->private; 700 /* clear private here, it is used to chain pages */ 701 p->private = 0; 702 } else 703 p = alloc_page(gfp_mask); 704 return p; 705} 706 707static void virtnet_rq_free_buf(struct virtnet_info *vi, 708 struct receive_queue *rq, void *buf) 709{ 710 if (vi->mergeable_rx_bufs) 711 put_page(virt_to_head_page(buf)); 712 else if (vi->big_packets) 713 give_pages(rq, buf); 714 else 715 put_page(virt_to_head_page(buf)); 716} 717 718static void enable_rx_mode_work(struct virtnet_info *vi) 719{ 720 rtnl_lock(); 721 vi->rx_mode_work_enabled = true; 722 rtnl_unlock(); 723} 724 725static void disable_rx_mode_work(struct virtnet_info *vi) 726{ 727 rtnl_lock(); 728 vi->rx_mode_work_enabled = false; 729 rtnl_unlock(); 730} 731 732static void virtqueue_napi_schedule(struct napi_struct *napi, 733 struct virtqueue *vq) 734{ 735 if (napi_schedule_prep(napi)) { 736 virtqueue_disable_cb(vq); 737 __napi_schedule(napi); 738 } 739} 740 741static bool virtqueue_napi_complete(struct napi_struct *napi, 742 struct virtqueue *vq, int processed) 743{ 744 int opaque; 745 746 opaque = virtqueue_enable_cb_prepare(vq); 747 if (napi_complete_done(napi, processed)) { 748 if (unlikely(virtqueue_poll(vq, opaque))) 749 virtqueue_napi_schedule(napi, vq); 750 else 751 return true; 752 } else { 753 virtqueue_disable_cb(vq); 754 } 755 756 return false; 757} 758 759static void virtnet_tx_wake_queue(struct virtnet_info *vi, 760 struct send_queue *sq) 761{ 762 unsigned int index = vq2txq(sq->vq); 763 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 764 765 if (netif_tx_queue_stopped(txq)) { 766 u64_stats_update_begin(&sq->stats.syncp); 767 u64_stats_inc(&sq->stats.wake); 768 u64_stats_update_end(&sq->stats.syncp); 769 netif_tx_wake_queue(txq); 770 } 771} 772 773static void skb_xmit_done(struct virtqueue *vq) 774{ 775 struct virtnet_info *vi = vq->vdev->priv; 776 unsigned int index = vq2txq(vq); 777 struct send_queue *sq = &vi->sq[index]; 778 struct napi_struct *napi = &sq->napi; 779 780 /* Suppress further interrupts. */ 781 virtqueue_disable_cb(vq); 782 783 if (napi->weight) 784 virtqueue_napi_schedule(napi, vq); 785 else 786 virtnet_tx_wake_queue(vi, sq); 787} 788 789#define MRG_CTX_HEADER_SHIFT 22 790static void *mergeable_len_to_ctx(unsigned int truesize, 791 unsigned int headroom) 792{ 793 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 794} 795 796static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 797{ 798 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 799} 800 801static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 802{ 803 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 804} 805 806static int check_mergeable_len(struct net_device *dev, void *mrg_ctx, 807 unsigned int len) 808{ 809 unsigned int headroom, tailroom, room, truesize; 810 811 truesize = mergeable_ctx_to_truesize(mrg_ctx); 812 headroom = mergeable_ctx_to_headroom(mrg_ctx); 813 tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 814 room = SKB_DATA_ALIGN(headroom + tailroom); 815 816 if (len > truesize - room) { 817 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 818 dev->name, len, (unsigned long)(truesize - room)); 819 DEV_STATS_INC(dev, rx_length_errors); 820 return -1; 821 } 822 823 return 0; 824} 825 826static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen, 827 unsigned int headroom, 828 unsigned int len) 829{ 830 struct sk_buff *skb; 831 832 skb = build_skb(buf, buflen); 833 if (unlikely(!skb)) 834 return NULL; 835 836 skb_reserve(skb, headroom); 837 skb_put(skb, len); 838 839 return skb; 840} 841 842/* Called from bottom half context */ 843static struct sk_buff *page_to_skb(struct virtnet_info *vi, 844 struct receive_queue *rq, 845 struct page *page, unsigned int offset, 846 unsigned int len, unsigned int truesize, 847 unsigned int headroom) 848{ 849 struct sk_buff *skb; 850 struct virtio_net_common_hdr *hdr; 851 unsigned int copy, hdr_len, hdr_padded_len; 852 struct page *page_to_free = NULL; 853 int tailroom, shinfo_size; 854 char *p, *hdr_p, *buf; 855 856 p = page_address(page) + offset; 857 hdr_p = p; 858 859 hdr_len = vi->hdr_len; 860 if (vi->mergeable_rx_bufs) 861 hdr_padded_len = hdr_len; 862 else 863 hdr_padded_len = sizeof(struct padded_vnet_hdr); 864 865 buf = p - headroom; 866 len -= hdr_len; 867 offset += hdr_padded_len; 868 p += hdr_padded_len; 869 tailroom = truesize - headroom - hdr_padded_len - len; 870 871 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 872 873 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 874 skb = virtnet_build_skb(buf, truesize, p - buf, len); 875 if (unlikely(!skb)) 876 return NULL; 877 878 page = (struct page *)page->private; 879 if (page) 880 give_pages(rq, page); 881 goto ok; 882 } 883 884 /* copy small packet so we can reuse these pages for small data */ 885 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 886 if (unlikely(!skb)) 887 return NULL; 888 889 /* Copy all frame if it fits skb->head, otherwise 890 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 891 */ 892 if (len <= skb_tailroom(skb)) 893 copy = len; 894 else 895 copy = ETH_HLEN; 896 skb_put_data(skb, p, copy); 897 898 len -= copy; 899 offset += copy; 900 901 if (vi->mergeable_rx_bufs) { 902 if (len) 903 skb_add_rx_frag(skb, 0, page, offset, len, truesize); 904 else 905 page_to_free = page; 906 goto ok; 907 } 908 909 BUG_ON(offset >= PAGE_SIZE); 910 while (len) { 911 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 912 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 913 frag_size, truesize); 914 len -= frag_size; 915 page = (struct page *)page->private; 916 offset = 0; 917 } 918 919 if (page) 920 give_pages(rq, page); 921 922ok: 923 hdr = skb_vnet_common_hdr(skb); 924 memcpy(hdr, hdr_p, hdr_len); 925 if (page_to_free) 926 put_page(page_to_free); 927 928 return skb; 929} 930 931static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len) 932{ 933 struct virtnet_info *vi = rq->vq->vdev->priv; 934 struct page *page = virt_to_head_page(buf); 935 struct virtnet_rq_dma *dma; 936 void *head; 937 int offset; 938 939 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 940 941 head = page_address(page); 942 943 dma = head; 944 945 --dma->ref; 946 947 if (dma->need_sync && len) { 948 offset = buf - (head + sizeof(*dma)); 949 950 virtqueue_map_sync_single_range_for_cpu(rq->vq, dma->addr, 951 offset, len, 952 DMA_FROM_DEVICE); 953 } 954 955 if (dma->ref) 956 return; 957 958 virtqueue_unmap_single_attrs(rq->vq, dma->addr, dma->len, 959 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 960 put_page(page); 961} 962 963static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx) 964{ 965 struct virtnet_info *vi = rq->vq->vdev->priv; 966 void *buf; 967 968 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 969 970 buf = virtqueue_get_buf_ctx(rq->vq, len, ctx); 971 if (buf) 972 virtnet_rq_unmap(rq, buf, *len); 973 974 return buf; 975} 976 977static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len) 978{ 979 struct virtnet_info *vi = rq->vq->vdev->priv; 980 struct virtnet_rq_dma *dma; 981 dma_addr_t addr; 982 u32 offset; 983 void *head; 984 985 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 986 987 head = page_address(rq->alloc_frag.page); 988 989 offset = buf - head; 990 991 dma = head; 992 993 addr = dma->addr - sizeof(*dma) + offset; 994 995 sg_init_table(rq->sg, 1); 996 sg_fill_dma(rq->sg, addr, len); 997} 998 999static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp) 1000{ 1001 struct page_frag *alloc_frag = &rq->alloc_frag; 1002 struct virtnet_info *vi = rq->vq->vdev->priv; 1003 struct virtnet_rq_dma *dma; 1004 void *buf, *head; 1005 dma_addr_t addr; 1006 1007 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 1008 1009 head = page_address(alloc_frag->page); 1010 1011 dma = head; 1012 1013 /* new pages */ 1014 if (!alloc_frag->offset) { 1015 if (rq->last_dma) { 1016 /* Now, the new page is allocated, the last dma 1017 * will not be used. So the dma can be unmapped 1018 * if the ref is 0. 1019 */ 1020 virtnet_rq_unmap(rq, rq->last_dma, 0); 1021 rq->last_dma = NULL; 1022 } 1023 1024 dma->len = alloc_frag->size - sizeof(*dma); 1025 1026 addr = virtqueue_map_single_attrs(rq->vq, dma + 1, 1027 dma->len, DMA_FROM_DEVICE, 0); 1028 if (virtqueue_map_mapping_error(rq->vq, addr)) 1029 return NULL; 1030 1031 dma->addr = addr; 1032 dma->need_sync = virtqueue_map_need_sync(rq->vq, addr); 1033 1034 /* Add a reference to dma to prevent the entire dma from 1035 * being released during error handling. This reference 1036 * will be freed after the pages are no longer used. 1037 */ 1038 get_page(alloc_frag->page); 1039 dma->ref = 1; 1040 alloc_frag->offset = sizeof(*dma); 1041 1042 rq->last_dma = dma; 1043 } 1044 1045 ++dma->ref; 1046 1047 buf = head + alloc_frag->offset; 1048 1049 get_page(alloc_frag->page); 1050 alloc_frag->offset += size; 1051 1052 return buf; 1053} 1054 1055static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf) 1056{ 1057 struct virtnet_info *vi = vq->vdev->priv; 1058 struct receive_queue *rq; 1059 int i = vq2rxq(vq); 1060 1061 rq = &vi->rq[i]; 1062 1063 if (rq->xsk_pool) { 1064 xsk_buff_free((struct xdp_buff *)buf); 1065 return; 1066 } 1067 1068 if (!vi->big_packets || vi->mergeable_rx_bufs) 1069 virtnet_rq_unmap(rq, buf, 0); 1070 1071 virtnet_rq_free_buf(vi, rq, buf); 1072} 1073 1074static void free_old_xmit(struct send_queue *sq, struct netdev_queue *txq, 1075 bool in_napi) 1076{ 1077 struct virtnet_sq_free_stats stats = {0}; 1078 1079 virtnet_free_old_xmit(sq, txq, in_napi, &stats); 1080 1081 /* Avoid overhead when no packets have been processed 1082 * happens when called speculatively from start_xmit. 1083 */ 1084 if (!stats.packets && !stats.napi_packets) 1085 return; 1086 1087 u64_stats_update_begin(&sq->stats.syncp); 1088 u64_stats_add(&sq->stats.bytes, stats.bytes + stats.napi_bytes); 1089 u64_stats_add(&sq->stats.packets, stats.packets + stats.napi_packets); 1090 u64_stats_update_end(&sq->stats.syncp); 1091} 1092 1093static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 1094{ 1095 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 1096 return false; 1097 else if (q < vi->curr_queue_pairs) 1098 return true; 1099 else 1100 return false; 1101} 1102 1103static bool tx_may_stop(struct virtnet_info *vi, 1104 struct net_device *dev, 1105 struct send_queue *sq) 1106{ 1107 int qnum; 1108 1109 qnum = sq - vi->sq; 1110 1111 /* If running out of space, stop queue to avoid getting packets that we 1112 * are then unable to transmit. 1113 * An alternative would be to force queuing layer to requeue the skb by 1114 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 1115 * returned in a normal path of operation: it means that driver is not 1116 * maintaining the TX queue stop/start state properly, and causes 1117 * the stack to do a non-trivial amount of useless work. 1118 * Since most packets only take 1 or 2 ring slots, stopping the queue 1119 * early means 16 slots are typically wasted. 1120 */ 1121 if (sq->vq->num_free < MAX_SKB_FRAGS + 2) { 1122 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 1123 1124 netif_tx_stop_queue(txq); 1125 u64_stats_update_begin(&sq->stats.syncp); 1126 u64_stats_inc(&sq->stats.stop); 1127 u64_stats_update_end(&sq->stats.syncp); 1128 1129 return true; 1130 } 1131 1132 return false; 1133} 1134 1135static void check_sq_full_and_disable(struct virtnet_info *vi, 1136 struct net_device *dev, 1137 struct send_queue *sq) 1138{ 1139 bool use_napi = sq->napi.weight; 1140 int qnum; 1141 1142 qnum = sq - vi->sq; 1143 1144 if (tx_may_stop(vi, dev, sq)) { 1145 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 1146 1147 if (use_napi) { 1148 if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 1149 virtqueue_napi_schedule(&sq->napi, sq->vq); 1150 } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 1151 /* More just got used, free them then recheck. */ 1152 free_old_xmit(sq, txq, false); 1153 if (sq->vq->num_free >= MAX_SKB_FRAGS + 2) { 1154 netif_start_subqueue(dev, qnum); 1155 u64_stats_update_begin(&sq->stats.syncp); 1156 u64_stats_inc(&sq->stats.wake); 1157 u64_stats_update_end(&sq->stats.syncp); 1158 virtqueue_disable_cb(sq->vq); 1159 } 1160 } 1161 } 1162} 1163 1164/* Note that @len is the length of received data without virtio header */ 1165static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi, 1166 struct receive_queue *rq, void *buf, 1167 u32 len, bool first_buf) 1168{ 1169 struct xdp_buff *xdp; 1170 u32 bufsize; 1171 1172 xdp = (struct xdp_buff *)buf; 1173 1174 /* In virtnet_add_recvbuf_xsk, we use part of XDP_PACKET_HEADROOM for 1175 * virtio header and ask the vhost to fill data from 1176 * hard_start + XDP_PACKET_HEADROOM - vi->hdr_len 1177 * The first buffer has virtio header so the remaining region for frame 1178 * data is 1179 * xsk_pool_get_rx_frame_size() 1180 * While other buffers than the first one do not have virtio header, so 1181 * the maximum frame data's length can be 1182 * xsk_pool_get_rx_frame_size() + vi->hdr_len 1183 */ 1184 bufsize = xsk_pool_get_rx_frame_size(rq->xsk_pool); 1185 if (!first_buf) 1186 bufsize += vi->hdr_len; 1187 1188 if (unlikely(len > bufsize)) { 1189 pr_debug("%s: rx error: len %u exceeds truesize %u\n", 1190 vi->dev->name, len, bufsize); 1191 DEV_STATS_INC(vi->dev, rx_length_errors); 1192 xsk_buff_free(xdp); 1193 return NULL; 1194 } 1195 1196 if (first_buf) { 1197 xsk_buff_set_size(xdp, len); 1198 } else { 1199 xdp_prepare_buff(xdp, xdp->data_hard_start, 1200 XDP_PACKET_HEADROOM - vi->hdr_len, len, 1); 1201 xdp->flags = 0; 1202 } 1203 1204 xsk_buff_dma_sync_for_cpu(xdp); 1205 1206 return xdp; 1207} 1208 1209static struct sk_buff *xsk_construct_skb(struct receive_queue *rq, 1210 struct xdp_buff *xdp) 1211{ 1212 unsigned int metasize = xdp->data - xdp->data_meta; 1213 struct sk_buff *skb; 1214 unsigned int size; 1215 1216 size = xdp->data_end - xdp->data_hard_start; 1217 skb = napi_alloc_skb(&rq->napi, size); 1218 if (unlikely(!skb)) { 1219 xsk_buff_free(xdp); 1220 return NULL; 1221 } 1222 1223 skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); 1224 1225 size = xdp->data_end - xdp->data_meta; 1226 memcpy(__skb_put(skb, size), xdp->data_meta, size); 1227 1228 if (metasize) { 1229 __skb_pull(skb, metasize); 1230 skb_metadata_set(skb, metasize); 1231 } 1232 1233 xsk_buff_free(xdp); 1234 1235 return skb; 1236} 1237 1238static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi, 1239 struct receive_queue *rq, struct xdp_buff *xdp, 1240 unsigned int *xdp_xmit, 1241 struct virtnet_rq_stats *stats) 1242{ 1243 struct bpf_prog *prog; 1244 u32 ret; 1245 1246 ret = XDP_PASS; 1247 rcu_read_lock(); 1248 prog = rcu_dereference(rq->xdp_prog); 1249 if (prog) 1250 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats); 1251 rcu_read_unlock(); 1252 1253 switch (ret) { 1254 case XDP_PASS: 1255 return xsk_construct_skb(rq, xdp); 1256 1257 case XDP_TX: 1258 case XDP_REDIRECT: 1259 return NULL; 1260 1261 default: 1262 /* drop packet */ 1263 xsk_buff_free(xdp); 1264 u64_stats_inc(&stats->drops); 1265 return NULL; 1266 } 1267} 1268 1269static void xsk_drop_follow_bufs(struct net_device *dev, 1270 struct receive_queue *rq, 1271 u32 num_buf, 1272 struct virtnet_rq_stats *stats) 1273{ 1274 struct xdp_buff *xdp; 1275 u32 len; 1276 1277 while (num_buf-- > 1) { 1278 xdp = virtqueue_get_buf(rq->vq, &len); 1279 if (unlikely(!xdp)) { 1280 pr_debug("%s: rx error: %d buffers missing\n", 1281 dev->name, num_buf); 1282 DEV_STATS_INC(dev, rx_length_errors); 1283 break; 1284 } 1285 u64_stats_add(&stats->bytes, len); 1286 xsk_buff_free(xdp); 1287 } 1288} 1289 1290static int xsk_append_merge_buffer(struct virtnet_info *vi, 1291 struct receive_queue *rq, 1292 struct sk_buff *head_skb, 1293 u32 num_buf, 1294 struct virtio_net_hdr_mrg_rxbuf *hdr, 1295 struct virtnet_rq_stats *stats) 1296{ 1297 struct sk_buff *curr_skb; 1298 struct xdp_buff *xdp; 1299 u32 len, truesize; 1300 struct page *page; 1301 void *buf; 1302 1303 curr_skb = head_skb; 1304 1305 while (--num_buf) { 1306 buf = virtqueue_get_buf(rq->vq, &len); 1307 if (unlikely(!buf)) { 1308 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1309 vi->dev->name, num_buf, 1310 virtio16_to_cpu(vi->vdev, 1311 hdr->num_buffers)); 1312 DEV_STATS_INC(vi->dev, rx_length_errors); 1313 return -EINVAL; 1314 } 1315 1316 u64_stats_add(&stats->bytes, len); 1317 1318 xdp = buf_to_xdp(vi, rq, buf, len, false); 1319 if (!xdp) 1320 goto err; 1321 1322 buf = napi_alloc_frag(len); 1323 if (!buf) { 1324 xsk_buff_free(xdp); 1325 goto err; 1326 } 1327 1328 memcpy(buf, xdp->data, len); 1329 1330 xsk_buff_free(xdp); 1331 1332 page = virt_to_page(buf); 1333 1334 truesize = len; 1335 1336 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page, 1337 buf, len, truesize); 1338 if (!curr_skb) { 1339 put_page(page); 1340 goto err; 1341 } 1342 } 1343 1344 return 0; 1345 1346err: 1347 xsk_drop_follow_bufs(vi->dev, rq, num_buf, stats); 1348 return -EINVAL; 1349} 1350 1351static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi, 1352 struct receive_queue *rq, struct xdp_buff *xdp, 1353 unsigned int *xdp_xmit, 1354 struct virtnet_rq_stats *stats) 1355{ 1356 struct virtio_net_hdr_mrg_rxbuf *hdr; 1357 struct bpf_prog *prog; 1358 struct sk_buff *skb; 1359 u32 ret, num_buf; 1360 1361 hdr = xdp->data - vi->hdr_len; 1362 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 1363 1364 ret = XDP_PASS; 1365 rcu_read_lock(); 1366 prog = rcu_dereference(rq->xdp_prog); 1367 if (prog) { 1368 /* TODO: support multi buffer. */ 1369 if (num_buf == 1) 1370 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, 1371 stats); 1372 else 1373 ret = XDP_ABORTED; 1374 } 1375 rcu_read_unlock(); 1376 1377 switch (ret) { 1378 case XDP_PASS: 1379 skb = xsk_construct_skb(rq, xdp); 1380 if (!skb) 1381 goto drop_bufs; 1382 1383 if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) { 1384 dev_kfree_skb(skb); 1385 goto drop; 1386 } 1387 1388 return skb; 1389 1390 case XDP_TX: 1391 case XDP_REDIRECT: 1392 return NULL; 1393 1394 default: 1395 /* drop packet */ 1396 xsk_buff_free(xdp); 1397 } 1398 1399drop_bufs: 1400 xsk_drop_follow_bufs(dev, rq, num_buf, stats); 1401 1402drop: 1403 u64_stats_inc(&stats->drops); 1404 return NULL; 1405} 1406 1407static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq, 1408 void *buf, u32 len, 1409 unsigned int *xdp_xmit, 1410 struct virtnet_rq_stats *stats) 1411{ 1412 struct net_device *dev = vi->dev; 1413 struct sk_buff *skb = NULL; 1414 struct xdp_buff *xdp; 1415 u8 flags; 1416 1417 len -= vi->hdr_len; 1418 1419 u64_stats_add(&stats->bytes, len); 1420 1421 xdp = buf_to_xdp(vi, rq, buf, len, true); 1422 if (!xdp) 1423 return; 1424 1425 if (unlikely(len < ETH_HLEN)) { 1426 pr_debug("%s: short packet %i\n", dev->name, len); 1427 DEV_STATS_INC(dev, rx_length_errors); 1428 xsk_buff_free(xdp); 1429 return; 1430 } 1431 1432 flags = ((struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags; 1433 1434 if (!vi->mergeable_rx_bufs) 1435 skb = virtnet_receive_xsk_small(dev, vi, rq, xdp, xdp_xmit, stats); 1436 else 1437 skb = virtnet_receive_xsk_merge(dev, vi, rq, xdp, xdp_xmit, stats); 1438 1439 if (skb) 1440 virtnet_receive_done(vi, rq, skb, flags); 1441} 1442 1443static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq, 1444 struct xsk_buff_pool *pool, gfp_t gfp) 1445{ 1446 struct xdp_buff **xsk_buffs; 1447 dma_addr_t addr; 1448 int err = 0; 1449 u32 len, i; 1450 int num; 1451 1452 xsk_buffs = rq->xsk_buffs; 1453 1454 num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free); 1455 if (!num) 1456 return -ENOMEM; 1457 1458 len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len; 1459 1460 for (i = 0; i < num; ++i) { 1461 /* Use the part of XDP_PACKET_HEADROOM as the virtnet hdr space. 1462 * We assume XDP_PACKET_HEADROOM is larger than hdr->len. 1463 * (see function virtnet_xsk_pool_enable) 1464 */ 1465 addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len; 1466 1467 sg_init_table(rq->sg, 1); 1468 sg_fill_dma(rq->sg, addr, len); 1469 1470 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, 1471 xsk_buffs[i], NULL, gfp); 1472 if (err) 1473 goto err; 1474 } 1475 1476 return num; 1477 1478err: 1479 for (; i < num; ++i) 1480 xsk_buff_free(xsk_buffs[i]); 1481 1482 return err; 1483} 1484 1485static void *virtnet_xsk_to_ptr(u32 len) 1486{ 1487 unsigned long p; 1488 1489 p = len << VIRTIO_XSK_FLAG_OFFSET; 1490 1491 return virtnet_xmit_ptr_pack((void *)p, VIRTNET_XMIT_TYPE_XSK); 1492} 1493 1494static int virtnet_xsk_xmit_one(struct send_queue *sq, 1495 struct xsk_buff_pool *pool, 1496 struct xdp_desc *desc) 1497{ 1498 struct virtnet_info *vi; 1499 dma_addr_t addr; 1500 1501 vi = sq->vq->vdev->priv; 1502 1503 addr = xsk_buff_raw_get_dma(pool, desc->addr); 1504 xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len); 1505 1506 sg_init_table(sq->sg, 2); 1507 sg_fill_dma(sq->sg, sq->xsk_hdr_dma_addr, vi->hdr_len); 1508 sg_fill_dma(sq->sg + 1, addr, desc->len); 1509 1510 return virtqueue_add_outbuf_premapped(sq->vq, sq->sg, 2, 1511 virtnet_xsk_to_ptr(desc->len), 1512 GFP_ATOMIC); 1513} 1514 1515static int virtnet_xsk_xmit_batch(struct send_queue *sq, 1516 struct xsk_buff_pool *pool, 1517 unsigned int budget, 1518 u64 *kicks) 1519{ 1520 struct xdp_desc *descs = pool->tx_descs; 1521 bool kick = false; 1522 u32 nb_pkts, i; 1523 int err; 1524 1525 budget = min_t(u32, budget, sq->vq->num_free); 1526 1527 nb_pkts = xsk_tx_peek_release_desc_batch(pool, budget); 1528 if (!nb_pkts) 1529 return 0; 1530 1531 for (i = 0; i < nb_pkts; i++) { 1532 err = virtnet_xsk_xmit_one(sq, pool, &descs[i]); 1533 if (unlikely(err)) { 1534 xsk_tx_completed(sq->xsk_pool, nb_pkts - i); 1535 break; 1536 } 1537 1538 kick = true; 1539 } 1540 1541 if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 1542 (*kicks)++; 1543 1544 return i; 1545} 1546 1547static bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool, 1548 int budget) 1549{ 1550 struct virtnet_info *vi = sq->vq->vdev->priv; 1551 struct virtnet_sq_free_stats stats = {}; 1552 struct net_device *dev = vi->dev; 1553 u64 kicks = 0; 1554 int sent; 1555 1556 /* Avoid to wakeup napi meanless, so call __free_old_xmit instead of 1557 * free_old_xmit(). 1558 */ 1559 __free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), true, &stats); 1560 1561 if (stats.xsk) 1562 xsk_tx_completed(sq->xsk_pool, stats.xsk); 1563 1564 sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks); 1565 1566 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 1567 check_sq_full_and_disable(vi, vi->dev, sq); 1568 1569 if (sent) { 1570 struct netdev_queue *txq; 1571 1572 txq = netdev_get_tx_queue(vi->dev, sq - vi->sq); 1573 txq_trans_cond_update(txq); 1574 } 1575 1576 u64_stats_update_begin(&sq->stats.syncp); 1577 u64_stats_add(&sq->stats.packets, stats.packets); 1578 u64_stats_add(&sq->stats.bytes, stats.bytes); 1579 u64_stats_add(&sq->stats.kicks, kicks); 1580 u64_stats_add(&sq->stats.xdp_tx, sent); 1581 u64_stats_update_end(&sq->stats.syncp); 1582 1583 if (xsk_uses_need_wakeup(pool)) 1584 xsk_set_tx_need_wakeup(pool); 1585 1586 return sent; 1587} 1588 1589static void xsk_wakeup(struct send_queue *sq) 1590{ 1591 if (napi_if_scheduled_mark_missed(&sq->napi)) 1592 return; 1593 1594 local_bh_disable(); 1595 virtqueue_napi_schedule(&sq->napi, sq->vq); 1596 local_bh_enable(); 1597} 1598 1599static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag) 1600{ 1601 struct virtnet_info *vi = netdev_priv(dev); 1602 struct send_queue *sq; 1603 1604 if (!netif_running(dev)) 1605 return -ENETDOWN; 1606 1607 if (qid >= vi->curr_queue_pairs) 1608 return -EINVAL; 1609 1610 sq = &vi->sq[qid]; 1611 1612 xsk_wakeup(sq); 1613 return 0; 1614} 1615 1616static void virtnet_xsk_completed(struct send_queue *sq, int num) 1617{ 1618 xsk_tx_completed(sq->xsk_pool, num); 1619 1620 /* If this is called by rx poll, start_xmit and xdp xmit we should 1621 * wakeup the tx napi to consume the xsk tx queue, because the tx 1622 * interrupt may not be triggered. 1623 */ 1624 xsk_wakeup(sq); 1625} 1626 1627static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 1628 struct send_queue *sq, 1629 struct xdp_frame *xdpf) 1630{ 1631 struct virtio_net_hdr_mrg_rxbuf *hdr; 1632 struct skb_shared_info *shinfo; 1633 u8 nr_frags = 0; 1634 int err, i; 1635 1636 if (unlikely(xdpf->headroom < vi->hdr_len)) 1637 return -EOVERFLOW; 1638 1639 if (unlikely(xdp_frame_has_frags(xdpf))) { 1640 shinfo = xdp_get_shared_info_from_frame(xdpf); 1641 nr_frags = shinfo->nr_frags; 1642 } 1643 1644 /* In wrapping function virtnet_xdp_xmit(), we need to free 1645 * up the pending old buffers, where we need to calculate the 1646 * position of skb_shared_info in xdp_get_frame_len() and 1647 * xdp_return_frame(), which will involve to xdpf->data and 1648 * xdpf->headroom. Therefore, we need to update the value of 1649 * headroom synchronously here. 1650 */ 1651 xdpf->headroom -= vi->hdr_len; 1652 xdpf->data -= vi->hdr_len; 1653 /* Zero header and leave csum up to XDP layers */ 1654 hdr = xdpf->data; 1655 memset(hdr, 0, vi->hdr_len); 1656 xdpf->len += vi->hdr_len; 1657 1658 sg_init_table(sq->sg, nr_frags + 1); 1659 sg_set_buf(sq->sg, xdpf->data, xdpf->len); 1660 for (i = 0; i < nr_frags; i++) { 1661 skb_frag_t *frag = &shinfo->frags[i]; 1662 1663 sg_set_page(&sq->sg[i + 1], skb_frag_page(frag), 1664 skb_frag_size(frag), skb_frag_off(frag)); 1665 } 1666 1667 err = virtnet_add_outbuf(sq, nr_frags + 1, xdpf, VIRTNET_XMIT_TYPE_XDP); 1668 if (unlikely(err)) 1669 return -ENOSPC; /* Caller handle free/refcnt */ 1670 1671 return 0; 1672} 1673 1674/* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 1675 * the current cpu, so it does not need to be locked. 1676 * 1677 * Here we use marco instead of inline functions because we have to deal with 1678 * three issues at the same time: 1. the choice of sq. 2. judge and execute the 1679 * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 1680 * functions to perfectly solve these three problems at the same time. 1681 */ 1682#define virtnet_xdp_get_sq(vi) ({ \ 1683 int cpu = smp_processor_id(); \ 1684 struct netdev_queue *txq; \ 1685 typeof(vi) v = (vi); \ 1686 unsigned int qp; \ 1687 \ 1688 if (v->curr_queue_pairs > nr_cpu_ids) { \ 1689 qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 1690 qp += cpu; \ 1691 txq = netdev_get_tx_queue(v->dev, qp); \ 1692 __netif_tx_acquire(txq); \ 1693 } else { \ 1694 qp = cpu % v->curr_queue_pairs; \ 1695 txq = netdev_get_tx_queue(v->dev, qp); \ 1696 __netif_tx_lock(txq, cpu); \ 1697 } \ 1698 v->sq + qp; \ 1699}) 1700 1701#define virtnet_xdp_put_sq(vi, q) { \ 1702 struct netdev_queue *txq; \ 1703 typeof(vi) v = (vi); \ 1704 \ 1705 txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 1706 if (v->curr_queue_pairs > nr_cpu_ids) \ 1707 __netif_tx_release(txq); \ 1708 else \ 1709 __netif_tx_unlock(txq); \ 1710} 1711 1712static int virtnet_xdp_xmit(struct net_device *dev, 1713 int n, struct xdp_frame **frames, u32 flags) 1714{ 1715 struct virtnet_info *vi = netdev_priv(dev); 1716 struct virtnet_sq_free_stats stats = {0}; 1717 struct receive_queue *rq = vi->rq; 1718 struct bpf_prog *xdp_prog; 1719 struct send_queue *sq; 1720 int nxmit = 0; 1721 int kicks = 0; 1722 int ret; 1723 int i; 1724 1725 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 1726 * indicate XDP resources have been successfully allocated. 1727 */ 1728 xdp_prog = rcu_access_pointer(rq->xdp_prog); 1729 if (!xdp_prog) 1730 return -ENXIO; 1731 1732 sq = virtnet_xdp_get_sq(vi); 1733 1734 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 1735 ret = -EINVAL; 1736 goto out; 1737 } 1738 1739 /* Free up any pending old buffers before queueing new ones. */ 1740 virtnet_free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), 1741 false, &stats); 1742 1743 for (i = 0; i < n; i++) { 1744 struct xdp_frame *xdpf = frames[i]; 1745 1746 if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 1747 break; 1748 nxmit++; 1749 } 1750 ret = nxmit; 1751 1752 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 1753 check_sq_full_and_disable(vi, dev, sq); 1754 1755 if (flags & XDP_XMIT_FLUSH) { 1756 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 1757 kicks = 1; 1758 } 1759out: 1760 u64_stats_update_begin(&sq->stats.syncp); 1761 u64_stats_add(&sq->stats.bytes, stats.bytes); 1762 u64_stats_add(&sq->stats.packets, stats.packets); 1763 u64_stats_add(&sq->stats.xdp_tx, n); 1764 u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit); 1765 u64_stats_add(&sq->stats.kicks, kicks); 1766 u64_stats_update_end(&sq->stats.syncp); 1767 1768 virtnet_xdp_put_sq(vi, sq); 1769 return ret; 1770} 1771 1772static void put_xdp_frags(struct xdp_buff *xdp) 1773{ 1774 struct skb_shared_info *shinfo; 1775 struct page *xdp_page; 1776 int i; 1777 1778 if (xdp_buff_has_frags(xdp)) { 1779 shinfo = xdp_get_shared_info_from_buff(xdp); 1780 for (i = 0; i < shinfo->nr_frags; i++) { 1781 xdp_page = skb_frag_page(&shinfo->frags[i]); 1782 put_page(xdp_page); 1783 } 1784 } 1785} 1786 1787static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, 1788 struct net_device *dev, 1789 unsigned int *xdp_xmit, 1790 struct virtnet_rq_stats *stats) 1791{ 1792 struct xdp_frame *xdpf; 1793 int err; 1794 u32 act; 1795 1796 act = bpf_prog_run_xdp(xdp_prog, xdp); 1797 u64_stats_inc(&stats->xdp_packets); 1798 1799 switch (act) { 1800 case XDP_PASS: 1801 return act; 1802 1803 case XDP_TX: 1804 u64_stats_inc(&stats->xdp_tx); 1805 xdpf = xdp_convert_buff_to_frame(xdp); 1806 if (unlikely(!xdpf)) { 1807 netdev_dbg(dev, "convert buff to frame failed for xdp\n"); 1808 return XDP_DROP; 1809 } 1810 1811 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 1812 if (unlikely(!err)) { 1813 xdp_return_frame_rx_napi(xdpf); 1814 } else if (unlikely(err < 0)) { 1815 trace_xdp_exception(dev, xdp_prog, act); 1816 return XDP_DROP; 1817 } 1818 *xdp_xmit |= VIRTIO_XDP_TX; 1819 return act; 1820 1821 case XDP_REDIRECT: 1822 u64_stats_inc(&stats->xdp_redirects); 1823 err = xdp_do_redirect(dev, xdp, xdp_prog); 1824 if (err) 1825 return XDP_DROP; 1826 1827 *xdp_xmit |= VIRTIO_XDP_REDIR; 1828 return act; 1829 1830 default: 1831 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); 1832 fallthrough; 1833 case XDP_ABORTED: 1834 trace_xdp_exception(dev, xdp_prog, act); 1835 fallthrough; 1836 case XDP_DROP: 1837 return XDP_DROP; 1838 } 1839} 1840 1841static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 1842{ 1843 return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0; 1844} 1845 1846/* We copy the packet for XDP in the following cases: 1847 * 1848 * 1) Packet is scattered across multiple rx buffers. 1849 * 2) Headroom space is insufficient. 1850 * 1851 * This is inefficient but it's a temporary condition that 1852 * we hit right after XDP is enabled and until queue is refilled 1853 * with large buffers with sufficient headroom - so it should affect 1854 * at most queue size packets. 1855 * Afterwards, the conditions to enable 1856 * XDP should preclude the underlying device from sending packets 1857 * across multiple buffers (num_buf > 1), and we make sure buffers 1858 * have enough headroom. 1859 */ 1860static struct page *xdp_linearize_page(struct net_device *dev, 1861 struct receive_queue *rq, 1862 int *num_buf, 1863 struct page *p, 1864 int offset, 1865 int page_off, 1866 unsigned int *len) 1867{ 1868 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1869 struct page *page; 1870 1871 if (page_off + *len + tailroom > PAGE_SIZE) 1872 return NULL; 1873 1874 page = alloc_page(GFP_ATOMIC); 1875 if (!page) 1876 return NULL; 1877 1878 memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 1879 page_off += *len; 1880 1881 /* Only mergeable mode can go inside this while loop. In small mode, 1882 * *num_buf == 1, so it cannot go inside. 1883 */ 1884 while (--*num_buf) { 1885 unsigned int buflen; 1886 void *buf; 1887 void *ctx; 1888 int off; 1889 1890 buf = virtnet_rq_get_buf(rq, &buflen, &ctx); 1891 if (unlikely(!buf)) 1892 goto err_buf; 1893 1894 p = virt_to_head_page(buf); 1895 off = buf - page_address(p); 1896 1897 if (check_mergeable_len(dev, ctx, buflen)) { 1898 put_page(p); 1899 goto err_buf; 1900 } 1901 1902 /* guard against a misconfigured or uncooperative backend that 1903 * is sending packet larger than the MTU. 1904 */ 1905 if ((page_off + buflen + tailroom) > PAGE_SIZE) { 1906 put_page(p); 1907 goto err_buf; 1908 } 1909 1910 memcpy(page_address(page) + page_off, 1911 page_address(p) + off, buflen); 1912 page_off += buflen; 1913 put_page(p); 1914 } 1915 1916 /* Headroom does not contribute to packet length */ 1917 *len = page_off - XDP_PACKET_HEADROOM; 1918 return page; 1919err_buf: 1920 __free_pages(page, 0); 1921 return NULL; 1922} 1923 1924static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi, 1925 unsigned int xdp_headroom, 1926 void *buf, 1927 unsigned int len) 1928{ 1929 unsigned int header_offset; 1930 unsigned int headroom; 1931 unsigned int buflen; 1932 struct sk_buff *skb; 1933 1934 header_offset = VIRTNET_RX_PAD + xdp_headroom; 1935 headroom = vi->hdr_len + header_offset; 1936 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1937 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1938 1939 skb = virtnet_build_skb(buf, buflen, headroom, len); 1940 if (unlikely(!skb)) 1941 return NULL; 1942 1943 buf += header_offset; 1944 memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len); 1945 1946 return skb; 1947} 1948 1949static struct sk_buff *receive_small_xdp(struct net_device *dev, 1950 struct virtnet_info *vi, 1951 struct receive_queue *rq, 1952 struct bpf_prog *xdp_prog, 1953 void *buf, 1954 unsigned int xdp_headroom, 1955 unsigned int len, 1956 unsigned int *xdp_xmit, 1957 struct virtnet_rq_stats *stats) 1958{ 1959 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 1960 unsigned int headroom = vi->hdr_len + header_offset; 1961 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 1962 struct page *page = virt_to_head_page(buf); 1963 struct page *xdp_page; 1964 unsigned int buflen; 1965 struct xdp_buff xdp; 1966 struct sk_buff *skb; 1967 unsigned int metasize = 0; 1968 u32 act; 1969 1970 if (unlikely(hdr->hdr.gso_type)) 1971 goto err_xdp; 1972 1973 /* Partially checksummed packets must be dropped. */ 1974 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) 1975 goto err_xdp; 1976 1977 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1978 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1979 1980 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 1981 int offset = buf - page_address(page) + header_offset; 1982 unsigned int tlen = len + vi->hdr_len; 1983 int num_buf = 1; 1984 1985 xdp_headroom = virtnet_get_headroom(vi); 1986 header_offset = VIRTNET_RX_PAD + xdp_headroom; 1987 headroom = vi->hdr_len + header_offset; 1988 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1989 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1990 xdp_page = xdp_linearize_page(dev, rq, &num_buf, page, 1991 offset, header_offset, 1992 &tlen); 1993 if (!xdp_page) 1994 goto err_xdp; 1995 1996 buf = page_address(xdp_page); 1997 put_page(page); 1998 page = xdp_page; 1999 } 2000 2001 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 2002 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 2003 xdp_headroom, len, true); 2004 2005 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 2006 2007 switch (act) { 2008 case XDP_PASS: 2009 /* Recalculate length in case bpf program changed it */ 2010 len = xdp.data_end - xdp.data; 2011 metasize = xdp.data - xdp.data_meta; 2012 break; 2013 2014 case XDP_TX: 2015 case XDP_REDIRECT: 2016 goto xdp_xmit; 2017 2018 default: 2019 goto err_xdp; 2020 } 2021 2022 skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len); 2023 if (unlikely(!skb)) 2024 goto err; 2025 2026 if (metasize) 2027 skb_metadata_set(skb, metasize); 2028 2029 return skb; 2030 2031err_xdp: 2032 u64_stats_inc(&stats->xdp_drops); 2033err: 2034 u64_stats_inc(&stats->drops); 2035 put_page(page); 2036xdp_xmit: 2037 return NULL; 2038} 2039 2040static struct sk_buff *receive_small(struct net_device *dev, 2041 struct virtnet_info *vi, 2042 struct receive_queue *rq, 2043 void *buf, void *ctx, 2044 unsigned int len, 2045 unsigned int *xdp_xmit, 2046 struct virtnet_rq_stats *stats) 2047{ 2048 unsigned int xdp_headroom = (unsigned long)ctx; 2049 struct page *page = virt_to_head_page(buf); 2050 struct sk_buff *skb; 2051 2052 /* We passed the address of virtnet header to virtio-core, 2053 * so truncate the padding. 2054 */ 2055 buf -= VIRTNET_RX_PAD + xdp_headroom; 2056 2057 len -= vi->hdr_len; 2058 u64_stats_add(&stats->bytes, len); 2059 2060 if (unlikely(len > GOOD_PACKET_LEN)) { 2061 pr_debug("%s: rx error: len %u exceeds max size %d\n", 2062 dev->name, len, GOOD_PACKET_LEN); 2063 DEV_STATS_INC(dev, rx_length_errors); 2064 goto err; 2065 } 2066 2067 if (unlikely(vi->xdp_enabled)) { 2068 struct bpf_prog *xdp_prog; 2069 2070 rcu_read_lock(); 2071 xdp_prog = rcu_dereference(rq->xdp_prog); 2072 if (xdp_prog) { 2073 skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf, 2074 xdp_headroom, len, xdp_xmit, 2075 stats); 2076 rcu_read_unlock(); 2077 return skb; 2078 } 2079 rcu_read_unlock(); 2080 } 2081 2082 skb = receive_small_build_skb(vi, xdp_headroom, buf, len); 2083 if (likely(skb)) 2084 return skb; 2085 2086err: 2087 u64_stats_inc(&stats->drops); 2088 put_page(page); 2089 return NULL; 2090} 2091 2092static struct sk_buff *receive_big(struct net_device *dev, 2093 struct virtnet_info *vi, 2094 struct receive_queue *rq, 2095 void *buf, 2096 unsigned int len, 2097 struct virtnet_rq_stats *stats) 2098{ 2099 struct page *page = buf; 2100 struct sk_buff *skb; 2101 2102 /* Make sure that len does not exceed the size allocated in 2103 * add_recvbuf_big. 2104 */ 2105 if (unlikely(len > (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE)) { 2106 pr_debug("%s: rx error: len %u exceeds allocated size %lu\n", 2107 dev->name, len, 2108 (vi->big_packets_num_skbfrags + 1) * PAGE_SIZE); 2109 goto err; 2110 } 2111 2112 skb = page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); 2113 u64_stats_add(&stats->bytes, len - vi->hdr_len); 2114 if (unlikely(!skb)) 2115 goto err; 2116 2117 return skb; 2118 2119err: 2120 u64_stats_inc(&stats->drops); 2121 give_pages(rq, page); 2122 return NULL; 2123} 2124 2125static void mergeable_buf_free(struct receive_queue *rq, int num_buf, 2126 struct net_device *dev, 2127 struct virtnet_rq_stats *stats) 2128{ 2129 struct page *page; 2130 void *buf; 2131 int len; 2132 2133 while (num_buf-- > 1) { 2134 buf = virtnet_rq_get_buf(rq, &len, NULL); 2135 if (unlikely(!buf)) { 2136 pr_debug("%s: rx error: %d buffers missing\n", 2137 dev->name, num_buf); 2138 DEV_STATS_INC(dev, rx_length_errors); 2139 break; 2140 } 2141 u64_stats_add(&stats->bytes, len); 2142 page = virt_to_head_page(buf); 2143 put_page(page); 2144 } 2145} 2146 2147/* Why not use xdp_build_skb_from_frame() ? 2148 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in 2149 * virtio-net there are 2 points that do not match its requirements: 2150 * 1. The size of the prefilled buffer is not fixed before xdp is set. 2151 * 2. xdp_build_skb_from_frame() does more checks that we don't need, 2152 * like eth_type_trans() (which virtio-net does in receive_buf()). 2153 */ 2154static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev, 2155 struct virtnet_info *vi, 2156 struct xdp_buff *xdp, 2157 unsigned int xdp_frags_truesz) 2158{ 2159 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 2160 unsigned int headroom, data_len; 2161 struct sk_buff *skb; 2162 int metasize; 2163 u8 nr_frags; 2164 2165 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 2166 pr_debug("Error building skb as missing reserved tailroom for xdp"); 2167 return NULL; 2168 } 2169 2170 if (unlikely(xdp_buff_has_frags(xdp))) 2171 nr_frags = sinfo->nr_frags; 2172 2173 skb = build_skb(xdp->data_hard_start, xdp->frame_sz); 2174 if (unlikely(!skb)) 2175 return NULL; 2176 2177 headroom = xdp->data - xdp->data_hard_start; 2178 data_len = xdp->data_end - xdp->data; 2179 skb_reserve(skb, headroom); 2180 __skb_put(skb, data_len); 2181 2182 metasize = xdp->data - xdp->data_meta; 2183 metasize = metasize > 0 ? metasize : 0; 2184 if (metasize) 2185 skb_metadata_set(skb, metasize); 2186 2187 if (unlikely(xdp_buff_has_frags(xdp))) 2188 xdp_update_skb_frags_info(skb, nr_frags, sinfo->xdp_frags_size, 2189 xdp_frags_truesz, 2190 xdp_buff_get_skb_flags(xdp)); 2191 2192 return skb; 2193} 2194 2195/* TODO: build xdp in big mode */ 2196static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 2197 struct virtnet_info *vi, 2198 struct receive_queue *rq, 2199 struct xdp_buff *xdp, 2200 void *buf, 2201 unsigned int len, 2202 unsigned int frame_sz, 2203 int *num_buf, 2204 unsigned int *xdp_frags_truesize, 2205 struct virtnet_rq_stats *stats) 2206{ 2207 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2208 struct skb_shared_info *shinfo; 2209 unsigned int xdp_frags_truesz = 0; 2210 unsigned int truesize; 2211 struct page *page; 2212 skb_frag_t *frag; 2213 int offset; 2214 void *ctx; 2215 2216 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 2217 xdp_prepare_buff(xdp, buf - XDP_PACKET_HEADROOM, 2218 XDP_PACKET_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 2219 2220 if (!*num_buf) 2221 return 0; 2222 2223 if (*num_buf > 1) { 2224 /* If we want to build multi-buffer xdp, we need 2225 * to specify that the flags of xdp_buff have the 2226 * XDP_FLAGS_HAS_FRAG bit. 2227 */ 2228 if (!xdp_buff_has_frags(xdp)) 2229 xdp_buff_set_frags_flag(xdp); 2230 2231 shinfo = xdp_get_shared_info_from_buff(xdp); 2232 shinfo->nr_frags = 0; 2233 shinfo->xdp_frags_size = 0; 2234 } 2235 2236 if (*num_buf > MAX_SKB_FRAGS + 1) 2237 return -EINVAL; 2238 2239 while (--*num_buf > 0) { 2240 buf = virtnet_rq_get_buf(rq, &len, &ctx); 2241 if (unlikely(!buf)) { 2242 pr_debug("%s: rx error: %d buffers out of %d missing\n", 2243 dev->name, *num_buf, 2244 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 2245 DEV_STATS_INC(dev, rx_length_errors); 2246 goto err; 2247 } 2248 2249 u64_stats_add(&stats->bytes, len); 2250 page = virt_to_head_page(buf); 2251 offset = buf - page_address(page); 2252 2253 if (check_mergeable_len(dev, ctx, len)) { 2254 put_page(page); 2255 goto err; 2256 } 2257 2258 truesize = mergeable_ctx_to_truesize(ctx); 2259 xdp_frags_truesz += truesize; 2260 2261 frag = &shinfo->frags[shinfo->nr_frags++]; 2262 skb_frag_fill_page_desc(frag, page, offset, len); 2263 if (page_is_pfmemalloc(page)) 2264 xdp_buff_set_frag_pfmemalloc(xdp); 2265 2266 shinfo->xdp_frags_size += len; 2267 } 2268 2269 *xdp_frags_truesize = xdp_frags_truesz; 2270 return 0; 2271 2272err: 2273 put_xdp_frags(xdp); 2274 return -EINVAL; 2275} 2276 2277static void *mergeable_xdp_get_buf(struct virtnet_info *vi, 2278 struct receive_queue *rq, 2279 struct bpf_prog *xdp_prog, 2280 void *ctx, 2281 unsigned int *frame_sz, 2282 int *num_buf, 2283 struct page **page, 2284 int offset, 2285 unsigned int *len, 2286 struct virtio_net_hdr_mrg_rxbuf *hdr) 2287{ 2288 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 2289 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 2290 struct page *xdp_page; 2291 unsigned int xdp_room; 2292 2293 /* Transient failure which in theory could occur if 2294 * in-flight packets from before XDP was enabled reach 2295 * the receive path after XDP is loaded. 2296 */ 2297 if (unlikely(hdr->hdr.gso_type)) 2298 return NULL; 2299 2300 /* Partially checksummed packets must be dropped. */ 2301 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) 2302 return NULL; 2303 2304 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers 2305 * with headroom may add hole in truesize, which 2306 * make their length exceed PAGE_SIZE. So we disabled the 2307 * hole mechanism for xdp. See add_recvbuf_mergeable(). 2308 */ 2309 *frame_sz = truesize; 2310 2311 if (likely(headroom >= virtnet_get_headroom(vi) && 2312 (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) { 2313 return page_address(*page) + offset; 2314 } 2315 2316 /* This happens when headroom is not enough because 2317 * of the buffer was prefilled before XDP is set. 2318 * This should only happen for the first several packets. 2319 * In fact, vq reset can be used here to help us clean up 2320 * the prefilled buffers, but many existing devices do not 2321 * support it, and we don't want to bother users who are 2322 * using xdp normally. 2323 */ 2324 if (!xdp_prog->aux->xdp_has_frags) { 2325 /* linearize data for XDP */ 2326 xdp_page = xdp_linearize_page(vi->dev, rq, num_buf, 2327 *page, offset, 2328 XDP_PACKET_HEADROOM, 2329 len); 2330 if (!xdp_page) 2331 return NULL; 2332 } else { 2333 xdp_room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM + 2334 sizeof(struct skb_shared_info)); 2335 if (*len + xdp_room > PAGE_SIZE) 2336 return NULL; 2337 2338 xdp_page = alloc_page(GFP_ATOMIC); 2339 if (!xdp_page) 2340 return NULL; 2341 2342 memcpy(page_address(xdp_page) + XDP_PACKET_HEADROOM, 2343 page_address(*page) + offset, *len); 2344 } 2345 2346 *frame_sz = PAGE_SIZE; 2347 2348 put_page(*page); 2349 2350 *page = xdp_page; 2351 2352 return page_address(*page) + XDP_PACKET_HEADROOM; 2353} 2354 2355static struct sk_buff *receive_mergeable_xdp(struct net_device *dev, 2356 struct virtnet_info *vi, 2357 struct receive_queue *rq, 2358 struct bpf_prog *xdp_prog, 2359 void *buf, 2360 void *ctx, 2361 unsigned int len, 2362 unsigned int *xdp_xmit, 2363 struct virtnet_rq_stats *stats) 2364{ 2365 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2366 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 2367 struct page *page = virt_to_head_page(buf); 2368 int offset = buf - page_address(page); 2369 unsigned int xdp_frags_truesz = 0; 2370 struct sk_buff *head_skb; 2371 unsigned int frame_sz; 2372 struct xdp_buff xdp; 2373 void *data; 2374 u32 act; 2375 int err; 2376 2377 data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page, 2378 offset, &len, hdr); 2379 if (unlikely(!data)) 2380 goto err_xdp; 2381 2382 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz, 2383 &num_buf, &xdp_frags_truesz, stats); 2384 if (unlikely(err)) 2385 goto err_xdp; 2386 2387 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 2388 2389 switch (act) { 2390 case XDP_PASS: 2391 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz); 2392 if (unlikely(!head_skb)) 2393 break; 2394 return head_skb; 2395 2396 case XDP_TX: 2397 case XDP_REDIRECT: 2398 return NULL; 2399 2400 default: 2401 break; 2402 } 2403 2404 put_xdp_frags(&xdp); 2405 2406err_xdp: 2407 put_page(page); 2408 mergeable_buf_free(rq, num_buf, dev, stats); 2409 2410 u64_stats_inc(&stats->xdp_drops); 2411 u64_stats_inc(&stats->drops); 2412 return NULL; 2413} 2414 2415static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb, 2416 struct sk_buff *curr_skb, 2417 struct page *page, void *buf, 2418 int len, int truesize) 2419{ 2420 int num_skb_frags; 2421 int offset; 2422 2423 num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 2424 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 2425 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 2426 2427 if (unlikely(!nskb)) 2428 return NULL; 2429 2430 if (curr_skb == head_skb) 2431 skb_shinfo(curr_skb)->frag_list = nskb; 2432 else 2433 curr_skb->next = nskb; 2434 curr_skb = nskb; 2435 head_skb->truesize += nskb->truesize; 2436 num_skb_frags = 0; 2437 } 2438 2439 if (curr_skb != head_skb) { 2440 head_skb->data_len += len; 2441 head_skb->len += len; 2442 head_skb->truesize += truesize; 2443 } 2444 2445 offset = buf - page_address(page); 2446 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 2447 put_page(page); 2448 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 2449 len, truesize); 2450 } else { 2451 skb_add_rx_frag(curr_skb, num_skb_frags, page, 2452 offset, len, truesize); 2453 } 2454 2455 return curr_skb; 2456} 2457 2458static struct sk_buff *receive_mergeable(struct net_device *dev, 2459 struct virtnet_info *vi, 2460 struct receive_queue *rq, 2461 void *buf, 2462 void *ctx, 2463 unsigned int len, 2464 unsigned int *xdp_xmit, 2465 struct virtnet_rq_stats *stats) 2466{ 2467 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2468 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 2469 struct page *page = virt_to_head_page(buf); 2470 int offset = buf - page_address(page); 2471 struct sk_buff *head_skb, *curr_skb; 2472 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 2473 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 2474 2475 head_skb = NULL; 2476 u64_stats_add(&stats->bytes, len - vi->hdr_len); 2477 2478 if (check_mergeable_len(dev, ctx, len)) 2479 goto err_skb; 2480 2481 if (unlikely(vi->xdp_enabled)) { 2482 struct bpf_prog *xdp_prog; 2483 2484 rcu_read_lock(); 2485 xdp_prog = rcu_dereference(rq->xdp_prog); 2486 if (xdp_prog) { 2487 head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx, 2488 len, xdp_xmit, stats); 2489 rcu_read_unlock(); 2490 return head_skb; 2491 } 2492 rcu_read_unlock(); 2493 } 2494 2495 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom); 2496 curr_skb = head_skb; 2497 2498 if (unlikely(!curr_skb)) 2499 goto err_skb; 2500 while (--num_buf) { 2501 buf = virtnet_rq_get_buf(rq, &len, &ctx); 2502 if (unlikely(!buf)) { 2503 pr_debug("%s: rx error: %d buffers out of %d missing\n", 2504 dev->name, num_buf, 2505 virtio16_to_cpu(vi->vdev, 2506 hdr->num_buffers)); 2507 DEV_STATS_INC(dev, rx_length_errors); 2508 goto err_buf; 2509 } 2510 2511 u64_stats_add(&stats->bytes, len); 2512 page = virt_to_head_page(buf); 2513 2514 if (check_mergeable_len(dev, ctx, len)) 2515 goto err_skb; 2516 2517 truesize = mergeable_ctx_to_truesize(ctx); 2518 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page, 2519 buf, len, truesize); 2520 if (!curr_skb) 2521 goto err_skb; 2522 } 2523 2524 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 2525 return head_skb; 2526 2527err_skb: 2528 put_page(page); 2529 mergeable_buf_free(rq, num_buf, dev, stats); 2530 2531err_buf: 2532 u64_stats_inc(&stats->drops); 2533 dev_kfree_skb(head_skb); 2534 return NULL; 2535} 2536 2537static inline u32 2538virtio_net_hash_value(const struct virtio_net_hdr_v1_hash *hdr_hash) 2539{ 2540 return __le16_to_cpu(hdr_hash->hash_value_lo) | 2541 (__le16_to_cpu(hdr_hash->hash_value_hi) << 16); 2542} 2543 2544static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash, 2545 struct sk_buff *skb) 2546{ 2547 enum pkt_hash_types rss_hash_type; 2548 2549 if (!hdr_hash || !skb) 2550 return; 2551 2552 switch (__le16_to_cpu(hdr_hash->hash_report)) { 2553 case VIRTIO_NET_HASH_REPORT_TCPv4: 2554 case VIRTIO_NET_HASH_REPORT_UDPv4: 2555 case VIRTIO_NET_HASH_REPORT_TCPv6: 2556 case VIRTIO_NET_HASH_REPORT_UDPv6: 2557 case VIRTIO_NET_HASH_REPORT_TCPv6_EX: 2558 case VIRTIO_NET_HASH_REPORT_UDPv6_EX: 2559 rss_hash_type = PKT_HASH_TYPE_L4; 2560 break; 2561 case VIRTIO_NET_HASH_REPORT_IPv4: 2562 case VIRTIO_NET_HASH_REPORT_IPv6: 2563 case VIRTIO_NET_HASH_REPORT_IPv6_EX: 2564 rss_hash_type = PKT_HASH_TYPE_L3; 2565 break; 2566 case VIRTIO_NET_HASH_REPORT_NONE: 2567 default: 2568 rss_hash_type = PKT_HASH_TYPE_NONE; 2569 } 2570 skb_set_hash(skb, virtio_net_hash_value(hdr_hash), rss_hash_type); 2571} 2572 2573static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq, 2574 struct sk_buff *skb, u8 flags) 2575{ 2576 struct virtio_net_common_hdr *hdr; 2577 struct net_device *dev = vi->dev; 2578 2579 hdr = skb_vnet_common_hdr(skb); 2580 if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) 2581 virtio_skb_set_hash(&hdr->hash_v1_hdr, skb); 2582 2583 hdr->hdr.flags = flags; 2584 if (virtio_net_handle_csum_offload(skb, &hdr->hdr, vi->rx_tnl_csum)) { 2585 net_warn_ratelimited("%s: bad csum: flags: %x, gso_type: %x rx_tnl_csum %d\n", 2586 dev->name, hdr->hdr.flags, 2587 hdr->hdr.gso_type, vi->rx_tnl_csum); 2588 goto frame_err; 2589 } 2590 2591 if (virtio_net_hdr_tnl_to_skb(skb, &hdr->tnl_hdr, vi->rx_tnl, 2592 vi->rx_tnl_csum, 2593 virtio_is_little_endian(vi->vdev))) { 2594 net_warn_ratelimited("%s: bad gso: type: %x, size: %u, flags %x tunnel %d tnl csum %d\n", 2595 dev->name, hdr->hdr.gso_type, 2596 hdr->hdr.gso_size, hdr->hdr.flags, 2597 vi->rx_tnl, vi->rx_tnl_csum); 2598 goto frame_err; 2599 } 2600 2601 skb_record_rx_queue(skb, vq2rxq(rq->vq)); 2602 skb->protocol = eth_type_trans(skb, dev); 2603 pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 2604 ntohs(skb->protocol), skb->len, skb->pkt_type); 2605 2606 napi_gro_receive(&rq->napi, skb); 2607 return; 2608 2609frame_err: 2610 DEV_STATS_INC(dev, rx_frame_errors); 2611 dev_kfree_skb(skb); 2612} 2613 2614static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 2615 void *buf, unsigned int len, void **ctx, 2616 unsigned int *xdp_xmit, 2617 struct virtnet_rq_stats *stats) 2618{ 2619 struct net_device *dev = vi->dev; 2620 struct sk_buff *skb; 2621 u8 flags; 2622 2623 if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 2624 pr_debug("%s: short packet %i\n", dev->name, len); 2625 DEV_STATS_INC(dev, rx_length_errors); 2626 virtnet_rq_free_buf(vi, rq, buf); 2627 return; 2628 } 2629 2630 /* About the flags below: 2631 * 1. Save the flags early, as the XDP program might overwrite them. 2632 * These flags ensure packets marked as VIRTIO_NET_HDR_F_DATA_VALID 2633 * stay valid after XDP processing. 2634 * 2. XDP doesn't work with partially checksummed packets (refer to 2635 * virtnet_xdp_set()), so packets marked as 2636 * VIRTIO_NET_HDR_F_NEEDS_CSUM get dropped during XDP processing. 2637 */ 2638 2639 if (vi->mergeable_rx_bufs) { 2640 flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; 2641 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 2642 stats); 2643 } else if (vi->big_packets) { 2644 void *p = page_address((struct page *)buf); 2645 2646 flags = ((struct virtio_net_common_hdr *)p)->hdr.flags; 2647 skb = receive_big(dev, vi, rq, buf, len, stats); 2648 } else { 2649 flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; 2650 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 2651 } 2652 2653 if (unlikely(!skb)) 2654 return; 2655 2656 virtnet_receive_done(vi, rq, skb, flags); 2657} 2658 2659/* Unlike mergeable buffers, all buffers are allocated to the 2660 * same size, except for the headroom. For this reason we do 2661 * not need to use mergeable_len_to_ctx here - it is enough 2662 * to store the headroom as the context ignoring the truesize. 2663 */ 2664static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 2665 gfp_t gfp) 2666{ 2667 char *buf; 2668 unsigned int xdp_headroom = virtnet_get_headroom(vi); 2669 void *ctx = (void *)(unsigned long)xdp_headroom; 2670 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 2671 int err; 2672 2673 len = SKB_DATA_ALIGN(len) + 2674 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2675 2676 if (unlikely(!skb_page_frag_refill(len, &rq->alloc_frag, gfp))) 2677 return -ENOMEM; 2678 2679 buf = virtnet_rq_alloc(rq, len, gfp); 2680 if (unlikely(!buf)) 2681 return -ENOMEM; 2682 2683 buf += VIRTNET_RX_PAD + xdp_headroom; 2684 2685 virtnet_rq_init_one_sg(rq, buf, vi->hdr_len + GOOD_PACKET_LEN); 2686 2687 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp); 2688 if (err < 0) { 2689 virtnet_rq_unmap(rq, buf, 0); 2690 put_page(virt_to_head_page(buf)); 2691 } 2692 2693 return err; 2694} 2695 2696static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 2697 gfp_t gfp) 2698{ 2699 struct page *first, *list = NULL; 2700 char *p; 2701 int i, err, offset; 2702 2703 sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); 2704 2705 /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ 2706 for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { 2707 first = get_a_page(rq, gfp); 2708 if (!first) { 2709 if (list) 2710 give_pages(rq, list); 2711 return -ENOMEM; 2712 } 2713 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 2714 2715 /* chain new page in list head to match sg */ 2716 first->private = (unsigned long)list; 2717 list = first; 2718 } 2719 2720 first = get_a_page(rq, gfp); 2721 if (!first) { 2722 give_pages(rq, list); 2723 return -ENOMEM; 2724 } 2725 p = page_address(first); 2726 2727 /* rq->sg[0], rq->sg[1] share the same page */ 2728 /* a separated rq->sg[0] for header - required in case !any_header_sg */ 2729 sg_set_buf(&rq->sg[0], p, vi->hdr_len); 2730 2731 /* rq->sg[1] for data packet, from offset */ 2732 offset = sizeof(struct padded_vnet_hdr); 2733 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 2734 2735 /* chain first in list head */ 2736 first->private = (unsigned long)list; 2737 err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, 2738 first, gfp); 2739 if (err < 0) 2740 give_pages(rq, first); 2741 2742 return err; 2743} 2744 2745static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 2746 struct ewma_pkt_len *avg_pkt_len, 2747 unsigned int room) 2748{ 2749 struct virtnet_info *vi = rq->vq->vdev->priv; 2750 const size_t hdr_len = vi->hdr_len; 2751 unsigned int len; 2752 2753 if (room) 2754 return PAGE_SIZE - room; 2755 2756 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 2757 rq->min_buf_len, PAGE_SIZE - hdr_len); 2758 2759 return ALIGN(len, L1_CACHE_BYTES); 2760} 2761 2762static int add_recvbuf_mergeable(struct virtnet_info *vi, 2763 struct receive_queue *rq, gfp_t gfp) 2764{ 2765 struct page_frag *alloc_frag = &rq->alloc_frag; 2766 unsigned int headroom = virtnet_get_headroom(vi); 2767 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 2768 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 2769 unsigned int len, hole; 2770 void *ctx; 2771 char *buf; 2772 int err; 2773 2774 /* Extra tailroom is needed to satisfy XDP's assumption. This 2775 * means rx frags coalescing won't work, but consider we've 2776 * disabled GSO for XDP, it won't be a big issue. 2777 */ 2778 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 2779 2780 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 2781 return -ENOMEM; 2782 2783 if (!alloc_frag->offset && len + room + sizeof(struct virtnet_rq_dma) > alloc_frag->size) 2784 len -= sizeof(struct virtnet_rq_dma); 2785 2786 buf = virtnet_rq_alloc(rq, len + room, gfp); 2787 if (unlikely(!buf)) 2788 return -ENOMEM; 2789 2790 buf += headroom; /* advance address leaving hole at front of pkt */ 2791 hole = alloc_frag->size - alloc_frag->offset; 2792 if (hole < len + room) { 2793 /* To avoid internal fragmentation, if there is very likely not 2794 * enough space for another buffer, add the remaining space to 2795 * the current buffer. 2796 * XDP core assumes that frame_size of xdp_buff and the length 2797 * of the frag are PAGE_SIZE, so we disable the hole mechanism. 2798 */ 2799 if (!headroom) 2800 len += hole; 2801 alloc_frag->offset += hole; 2802 } 2803 2804 virtnet_rq_init_one_sg(rq, buf, len); 2805 2806 ctx = mergeable_len_to_ctx(len + room, headroom); 2807 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp); 2808 if (err < 0) { 2809 virtnet_rq_unmap(rq, buf, 0); 2810 put_page(virt_to_head_page(buf)); 2811 } 2812 2813 return err; 2814} 2815 2816/* 2817 * Returns false if we couldn't fill entirely (OOM). 2818 * 2819 * Normally run in the receive path, but can also be run from ndo_open 2820 * before we're receiving packets, or from refill_work which is 2821 * careful to disable receiving (using napi_disable). 2822 */ 2823static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 2824 gfp_t gfp) 2825{ 2826 int err; 2827 2828 if (rq->xsk_pool) { 2829 err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk_pool, gfp); 2830 goto kick; 2831 } 2832 2833 do { 2834 if (vi->mergeable_rx_bufs) 2835 err = add_recvbuf_mergeable(vi, rq, gfp); 2836 else if (vi->big_packets) 2837 err = add_recvbuf_big(vi, rq, gfp); 2838 else 2839 err = add_recvbuf_small(vi, rq, gfp); 2840 2841 if (err) 2842 break; 2843 } while (rq->vq->num_free); 2844 2845kick: 2846 if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 2847 unsigned long flags; 2848 2849 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 2850 u64_stats_inc(&rq->stats.kicks); 2851 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 2852 } 2853 2854 return err != -ENOMEM; 2855} 2856 2857static void skb_recv_done(struct virtqueue *rvq) 2858{ 2859 struct virtnet_info *vi = rvq->vdev->priv; 2860 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 2861 2862 rq->calls++; 2863 virtqueue_napi_schedule(&rq->napi, rvq); 2864} 2865 2866static void virtnet_napi_do_enable(struct virtqueue *vq, 2867 struct napi_struct *napi) 2868{ 2869 napi_enable(napi); 2870 2871 /* If all buffers were filled by other side before we napi_enabled, we 2872 * won't get another interrupt, so process any outstanding packets now. 2873 * Call local_bh_enable after to trigger softIRQ processing. 2874 */ 2875 local_bh_disable(); 2876 virtqueue_napi_schedule(napi, vq); 2877 local_bh_enable(); 2878} 2879 2880static void virtnet_napi_enable(struct receive_queue *rq) 2881{ 2882 struct virtnet_info *vi = rq->vq->vdev->priv; 2883 int qidx = vq2rxq(rq->vq); 2884 2885 virtnet_napi_do_enable(rq->vq, &rq->napi); 2886 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, &rq->napi); 2887} 2888 2889static void virtnet_napi_tx_enable(struct send_queue *sq) 2890{ 2891 struct virtnet_info *vi = sq->vq->vdev->priv; 2892 struct napi_struct *napi = &sq->napi; 2893 int qidx = vq2txq(sq->vq); 2894 2895 if (!napi->weight) 2896 return; 2897 2898 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 2899 * enable the feature if this is likely affine with the transmit path. 2900 */ 2901 if (!vi->affinity_hint_set) { 2902 napi->weight = 0; 2903 return; 2904 } 2905 2906 virtnet_napi_do_enable(sq->vq, napi); 2907 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, napi); 2908} 2909 2910static void virtnet_napi_tx_disable(struct send_queue *sq) 2911{ 2912 struct virtnet_info *vi = sq->vq->vdev->priv; 2913 struct napi_struct *napi = &sq->napi; 2914 int qidx = vq2txq(sq->vq); 2915 2916 if (napi->weight) { 2917 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, NULL); 2918 napi_disable(napi); 2919 } 2920} 2921 2922static void virtnet_napi_disable(struct receive_queue *rq) 2923{ 2924 struct virtnet_info *vi = rq->vq->vdev->priv; 2925 struct napi_struct *napi = &rq->napi; 2926 int qidx = vq2rxq(rq->vq); 2927 2928 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, NULL); 2929 napi_disable(napi); 2930} 2931 2932static int virtnet_receive_xsk_bufs(struct virtnet_info *vi, 2933 struct receive_queue *rq, 2934 int budget, 2935 unsigned int *xdp_xmit, 2936 struct virtnet_rq_stats *stats) 2937{ 2938 unsigned int len; 2939 int packets = 0; 2940 void *buf; 2941 2942 while (packets < budget) { 2943 buf = virtqueue_get_buf(rq->vq, &len); 2944 if (!buf) 2945 break; 2946 2947 virtnet_receive_xsk_buf(vi, rq, buf, len, xdp_xmit, stats); 2948 packets++; 2949 } 2950 2951 return packets; 2952} 2953 2954static int virtnet_receive_packets(struct virtnet_info *vi, 2955 struct receive_queue *rq, 2956 int budget, 2957 unsigned int *xdp_xmit, 2958 struct virtnet_rq_stats *stats) 2959{ 2960 unsigned int len; 2961 int packets = 0; 2962 void *buf; 2963 2964 if (!vi->big_packets || vi->mergeable_rx_bufs) { 2965 void *ctx; 2966 while (packets < budget && 2967 (buf = virtnet_rq_get_buf(rq, &len, &ctx))) { 2968 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, stats); 2969 packets++; 2970 } 2971 } else { 2972 while (packets < budget && 2973 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 2974 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, stats); 2975 packets++; 2976 } 2977 } 2978 2979 return packets; 2980} 2981 2982static int virtnet_receive(struct receive_queue *rq, int budget, 2983 unsigned int *xdp_xmit) 2984{ 2985 struct virtnet_info *vi = rq->vq->vdev->priv; 2986 struct virtnet_rq_stats stats = {}; 2987 int i, packets; 2988 2989 if (rq->xsk_pool) 2990 packets = virtnet_receive_xsk_bufs(vi, rq, budget, xdp_xmit, &stats); 2991 else 2992 packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats); 2993 2994 u64_stats_set(&stats.packets, packets); 2995 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 2996 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) 2997 /* We need to retry refilling in the next NAPI poll so 2998 * we must return budget to make sure the NAPI is 2999 * repolled. 3000 */ 3001 packets = budget; 3002 } 3003 3004 u64_stats_update_begin(&rq->stats.syncp); 3005 for (i = 0; i < ARRAY_SIZE(virtnet_rq_stats_desc); i++) { 3006 size_t offset = virtnet_rq_stats_desc[i].offset; 3007 u64_stats_t *item, *src; 3008 3009 item = (u64_stats_t *)((u8 *)&rq->stats + offset); 3010 src = (u64_stats_t *)((u8 *)&stats + offset); 3011 u64_stats_add(item, u64_stats_read(src)); 3012 } 3013 3014 u64_stats_add(&rq->stats.packets, u64_stats_read(&stats.packets)); 3015 u64_stats_add(&rq->stats.bytes, u64_stats_read(&stats.bytes)); 3016 3017 u64_stats_update_end(&rq->stats.syncp); 3018 3019 return packets; 3020} 3021 3022static void virtnet_poll_cleantx(struct receive_queue *rq, int budget) 3023{ 3024 struct virtnet_info *vi = rq->vq->vdev->priv; 3025 unsigned int index = vq2rxq(rq->vq); 3026 struct send_queue *sq = &vi->sq[index]; 3027 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 3028 3029 if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 3030 return; 3031 3032 if (__netif_tx_trylock(txq)) { 3033 if (sq->reset) { 3034 __netif_tx_unlock(txq); 3035 return; 3036 } 3037 3038 do { 3039 virtqueue_disable_cb(sq->vq); 3040 free_old_xmit(sq, txq, !!budget); 3041 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 3042 3043 if (sq->vq->num_free >= MAX_SKB_FRAGS + 2) 3044 virtnet_tx_wake_queue(vi, sq); 3045 3046 __netif_tx_unlock(txq); 3047 } 3048} 3049 3050static void virtnet_rx_dim_update(struct virtnet_info *vi, struct receive_queue *rq) 3051{ 3052 struct dim_sample cur_sample = {}; 3053 3054 if (!rq->packets_in_napi) 3055 return; 3056 3057 /* Don't need protection when fetching stats, since fetcher and 3058 * updater of the stats are in same context 3059 */ 3060 dim_update_sample(rq->calls, 3061 u64_stats_read(&rq->stats.packets), 3062 u64_stats_read(&rq->stats.bytes), 3063 &cur_sample); 3064 3065 net_dim(&rq->dim, &cur_sample); 3066 rq->packets_in_napi = 0; 3067} 3068 3069static int virtnet_poll(struct napi_struct *napi, int budget) 3070{ 3071 struct receive_queue *rq = 3072 container_of(napi, struct receive_queue, napi); 3073 struct virtnet_info *vi = rq->vq->vdev->priv; 3074 struct send_queue *sq; 3075 unsigned int received; 3076 unsigned int xdp_xmit = 0; 3077 bool napi_complete; 3078 3079 virtnet_poll_cleantx(rq, budget); 3080 3081 received = virtnet_receive(rq, budget, &xdp_xmit); 3082 rq->packets_in_napi += received; 3083 3084 if (xdp_xmit & VIRTIO_XDP_REDIR) 3085 xdp_do_flush(); 3086 3087 /* Out of packets? */ 3088 if (received < budget) { 3089 napi_complete = virtqueue_napi_complete(napi, rq->vq, received); 3090 /* Intentionally not taking dim_lock here. This may result in a 3091 * spurious net_dim call. But if that happens virtnet_rx_dim_work 3092 * will not act on the scheduled work. 3093 */ 3094 if (napi_complete && rq->dim_enabled) 3095 virtnet_rx_dim_update(vi, rq); 3096 } 3097 3098 if (xdp_xmit & VIRTIO_XDP_TX) { 3099 sq = virtnet_xdp_get_sq(vi); 3100 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 3101 u64_stats_update_begin(&sq->stats.syncp); 3102 u64_stats_inc(&sq->stats.kicks); 3103 u64_stats_update_end(&sq->stats.syncp); 3104 } 3105 virtnet_xdp_put_sq(vi, sq); 3106 } 3107 3108 return received; 3109} 3110 3111static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index) 3112{ 3113 virtnet_napi_tx_disable(&vi->sq[qp_index]); 3114 virtnet_napi_disable(&vi->rq[qp_index]); 3115 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 3116} 3117 3118static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index) 3119{ 3120 struct net_device *dev = vi->dev; 3121 int err; 3122 3123 err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index, 3124 vi->rq[qp_index].napi.napi_id); 3125 if (err < 0) 3126 return err; 3127 3128 err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq, 3129 MEM_TYPE_PAGE_SHARED, NULL); 3130 if (err < 0) 3131 goto err_xdp_reg_mem_model; 3132 3133 virtnet_napi_enable(&vi->rq[qp_index]); 3134 virtnet_napi_tx_enable(&vi->sq[qp_index]); 3135 3136 return 0; 3137 3138err_xdp_reg_mem_model: 3139 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 3140 return err; 3141} 3142 3143static void virtnet_cancel_dim(struct virtnet_info *vi, struct dim *dim) 3144{ 3145 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 3146 return; 3147 net_dim_work_cancel(dim); 3148} 3149 3150static void virtnet_update_settings(struct virtnet_info *vi) 3151{ 3152 u32 speed; 3153 u8 duplex; 3154 3155 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 3156 return; 3157 3158 virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 3159 3160 if (ethtool_validate_speed(speed)) 3161 vi->speed = speed; 3162 3163 virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 3164 3165 if (ethtool_validate_duplex(duplex)) 3166 vi->duplex = duplex; 3167} 3168 3169static int virtnet_open(struct net_device *dev) 3170{ 3171 struct virtnet_info *vi = netdev_priv(dev); 3172 int i, err; 3173 3174 for (i = 0; i < vi->max_queue_pairs; i++) { 3175 if (i < vi->curr_queue_pairs) 3176 /* Pre-fill rq agressively, to make sure we are ready to 3177 * get packets immediately. 3178 */ 3179 try_fill_recv(vi, &vi->rq[i], GFP_KERNEL); 3180 3181 err = virtnet_enable_queue_pair(vi, i); 3182 if (err < 0) 3183 goto err_enable_qp; 3184 } 3185 3186 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 3187 if (vi->status & VIRTIO_NET_S_LINK_UP) 3188 netif_carrier_on(vi->dev); 3189 virtio_config_driver_enable(vi->vdev); 3190 } else { 3191 vi->status = VIRTIO_NET_S_LINK_UP; 3192 netif_carrier_on(dev); 3193 } 3194 3195 return 0; 3196 3197err_enable_qp: 3198 for (i--; i >= 0; i--) { 3199 virtnet_disable_queue_pair(vi, i); 3200 virtnet_cancel_dim(vi, &vi->rq[i].dim); 3201 } 3202 3203 return err; 3204} 3205 3206static int virtnet_poll_tx(struct napi_struct *napi, int budget) 3207{ 3208 struct send_queue *sq = container_of(napi, struct send_queue, napi); 3209 struct virtnet_info *vi = sq->vq->vdev->priv; 3210 unsigned int index = vq2txq(sq->vq); 3211 struct netdev_queue *txq; 3212 int opaque, xsk_done = 0; 3213 bool done; 3214 3215 if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 3216 /* We don't need to enable cb for XDP */ 3217 napi_complete_done(napi, 0); 3218 return 0; 3219 } 3220 3221 txq = netdev_get_tx_queue(vi->dev, index); 3222 __netif_tx_lock(txq, raw_smp_processor_id()); 3223 virtqueue_disable_cb(sq->vq); 3224 3225 if (sq->xsk_pool) 3226 xsk_done = virtnet_xsk_xmit(sq, sq->xsk_pool, budget); 3227 else 3228 free_old_xmit(sq, txq, !!budget); 3229 3230 if (sq->vq->num_free >= MAX_SKB_FRAGS + 2) 3231 virtnet_tx_wake_queue(vi, sq); 3232 3233 if (xsk_done >= budget) { 3234 __netif_tx_unlock(txq); 3235 return budget; 3236 } 3237 3238 opaque = virtqueue_enable_cb_prepare(sq->vq); 3239 3240 done = napi_complete_done(napi, 0); 3241 3242 if (!done) 3243 virtqueue_disable_cb(sq->vq); 3244 3245 __netif_tx_unlock(txq); 3246 3247 if (done) { 3248 if (unlikely(virtqueue_poll(sq->vq, opaque))) { 3249 if (napi_schedule_prep(napi)) { 3250 __netif_tx_lock(txq, raw_smp_processor_id()); 3251 virtqueue_disable_cb(sq->vq); 3252 __netif_tx_unlock(txq); 3253 __napi_schedule(napi); 3254 } 3255 } 3256 } 3257 3258 return 0; 3259} 3260 3261static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) 3262{ 3263 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 3264 struct virtnet_info *vi = sq->vq->vdev->priv; 3265 struct virtio_net_hdr_v1_hash_tunnel *hdr; 3266 int num_sg; 3267 unsigned hdr_len = vi->hdr_len; 3268 bool feature_hdrlen; 3269 bool can_push; 3270 3271 feature_hdrlen = virtio_has_feature(vi->vdev, 3272 VIRTIO_NET_F_GUEST_HDRLEN); 3273 3274 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 3275 3276 /* Make sure it's safe to cast between formats */ 3277 BUILD_BUG_ON(__alignof__(*hdr) != __alignof__(hdr->hash_hdr)); 3278 BUILD_BUG_ON(__alignof__(*hdr) != __alignof__(hdr->hash_hdr.hdr)); 3279 3280 can_push = vi->any_header_sg && 3281 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 3282 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 3283 /* Even if we can, don't push here yet as this would skew 3284 * csum_start offset below. */ 3285 if (can_push) 3286 hdr = (struct virtio_net_hdr_v1_hash_tunnel *)(skb->data - 3287 hdr_len); 3288 else 3289 hdr = &skb_vnet_common_hdr(skb)->tnl_hdr; 3290 3291 if (virtio_net_hdr_tnl_from_skb(skb, hdr, vi->tx_tnl, 3292 virtio_is_little_endian(vi->vdev), 0, 3293 false, feature_hdrlen)) 3294 return -EPROTO; 3295 3296 if (vi->mergeable_rx_bufs) 3297 hdr->hash_hdr.hdr.num_buffers = 0; 3298 3299 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 3300 if (can_push) { 3301 __skb_push(skb, hdr_len); 3302 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 3303 if (unlikely(num_sg < 0)) 3304 return num_sg; 3305 /* Pull header back to avoid skew in tx bytes calculations. */ 3306 __skb_pull(skb, hdr_len); 3307 } else { 3308 sg_set_buf(sq->sg, hdr, hdr_len); 3309 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 3310 if (unlikely(num_sg < 0)) 3311 return num_sg; 3312 num_sg++; 3313 } 3314 3315 return virtnet_add_outbuf(sq, num_sg, skb, 3316 orphan ? VIRTNET_XMIT_TYPE_SKB_ORPHAN : VIRTNET_XMIT_TYPE_SKB); 3317} 3318 3319static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 3320{ 3321 struct virtnet_info *vi = netdev_priv(dev); 3322 int qnum = skb_get_queue_mapping(skb); 3323 struct send_queue *sq = &vi->sq[qnum]; 3324 int err; 3325 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 3326 bool xmit_more = netdev_xmit_more(); 3327 bool use_napi = sq->napi.weight; 3328 bool kick; 3329 3330 if (!use_napi) 3331 free_old_xmit(sq, txq, false); 3332 else 3333 virtqueue_disable_cb(sq->vq); 3334 3335 /* timestamp packet in software */ 3336 skb_tx_timestamp(skb); 3337 3338 /* Try to transmit */ 3339 err = xmit_skb(sq, skb, !use_napi); 3340 3341 /* This should not happen! */ 3342 if (unlikely(err)) { 3343 DEV_STATS_INC(dev, tx_fifo_errors); 3344 if (net_ratelimit()) 3345 dev_warn(&dev->dev, 3346 "Unexpected TXQ (%d) queue failure: %d\n", 3347 qnum, err); 3348 DEV_STATS_INC(dev, tx_dropped); 3349 dev_kfree_skb_any(skb); 3350 return NETDEV_TX_OK; 3351 } 3352 3353 /* Don't wait up for transmitted skbs to be freed. */ 3354 if (!use_napi) { 3355 skb_orphan(skb); 3356 skb_dst_drop(skb); 3357 nf_reset_ct(skb); 3358 } 3359 3360 if (use_napi) 3361 tx_may_stop(vi, dev, sq); 3362 else 3363 check_sq_full_and_disable(vi, dev,sq); 3364 3365 kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) : 3366 !xmit_more || netif_xmit_stopped(txq); 3367 if (kick) { 3368 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 3369 u64_stats_update_begin(&sq->stats.syncp); 3370 u64_stats_inc(&sq->stats.kicks); 3371 u64_stats_update_end(&sq->stats.syncp); 3372 } 3373 } 3374 3375 if (use_napi && kick && unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 3376 virtqueue_napi_schedule(&sq->napi, sq->vq); 3377 3378 return NETDEV_TX_OK; 3379} 3380 3381static void virtnet_rx_pause(struct virtnet_info *vi, 3382 struct receive_queue *rq) 3383{ 3384 bool running = netif_running(vi->dev); 3385 3386 if (running) { 3387 virtnet_napi_disable(rq); 3388 virtnet_cancel_dim(vi, &rq->dim); 3389 } 3390} 3391 3392static void virtnet_rx_pause_all(struct virtnet_info *vi) 3393{ 3394 int i; 3395 3396 for (i = 0; i < vi->max_queue_pairs; i++) 3397 virtnet_rx_pause(vi, &vi->rq[i]); 3398} 3399 3400static void virtnet_rx_resume(struct virtnet_info *vi, 3401 struct receive_queue *rq, 3402 bool refill) 3403{ 3404 if (netif_running(vi->dev)) { 3405 /* Pre-fill rq agressively, to make sure we are ready to get 3406 * packets immediately. 3407 */ 3408 if (refill) 3409 try_fill_recv(vi, rq, GFP_KERNEL); 3410 3411 virtnet_napi_enable(rq); 3412 } 3413} 3414 3415static void virtnet_rx_resume_all(struct virtnet_info *vi) 3416{ 3417 int i; 3418 3419 for (i = 0; i < vi->max_queue_pairs; i++) { 3420 if (i < vi->curr_queue_pairs) 3421 virtnet_rx_resume(vi, &vi->rq[i], true); 3422 else 3423 virtnet_rx_resume(vi, &vi->rq[i], false); 3424 } 3425} 3426 3427static int virtnet_rx_resize(struct virtnet_info *vi, 3428 struct receive_queue *rq, u32 ring_num) 3429{ 3430 int err, qindex; 3431 3432 qindex = rq - vi->rq; 3433 3434 virtnet_rx_pause(vi, rq); 3435 3436 err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf, NULL); 3437 if (err) 3438 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); 3439 3440 virtnet_rx_resume(vi, rq, true); 3441 return err; 3442} 3443 3444static void virtnet_tx_pause(struct virtnet_info *vi, struct send_queue *sq) 3445{ 3446 bool running = netif_running(vi->dev); 3447 struct netdev_queue *txq; 3448 int qindex; 3449 3450 qindex = sq - vi->sq; 3451 3452 if (running) 3453 virtnet_napi_tx_disable(sq); 3454 3455 txq = netdev_get_tx_queue(vi->dev, qindex); 3456 3457 /* 1. wait all ximt complete 3458 * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue() 3459 */ 3460 __netif_tx_lock_bh(txq); 3461 3462 /* Prevent rx poll from accessing sq. */ 3463 sq->reset = true; 3464 3465 /* Prevent the upper layer from trying to send packets. */ 3466 netif_stop_subqueue(vi->dev, qindex); 3467 u64_stats_update_begin(&sq->stats.syncp); 3468 u64_stats_inc(&sq->stats.stop); 3469 u64_stats_update_end(&sq->stats.syncp); 3470 3471 __netif_tx_unlock_bh(txq); 3472} 3473 3474static void virtnet_tx_resume(struct virtnet_info *vi, struct send_queue *sq) 3475{ 3476 bool running = netif_running(vi->dev); 3477 struct netdev_queue *txq; 3478 int qindex; 3479 3480 qindex = sq - vi->sq; 3481 3482 txq = netdev_get_tx_queue(vi->dev, qindex); 3483 3484 __netif_tx_lock_bh(txq); 3485 sq->reset = false; 3486 virtnet_tx_wake_queue(vi, sq); 3487 __netif_tx_unlock_bh(txq); 3488 3489 if (running) 3490 virtnet_napi_tx_enable(sq); 3491} 3492 3493static int virtnet_tx_resize(struct virtnet_info *vi, struct send_queue *sq, 3494 u32 ring_num) 3495{ 3496 int qindex, err; 3497 3498 if (ring_num <= MAX_SKB_FRAGS + 2) { 3499 netdev_err(vi->dev, "tx size (%d) cannot be smaller than %d\n", 3500 ring_num, MAX_SKB_FRAGS + 2); 3501 return -EINVAL; 3502 } 3503 3504 qindex = sq - vi->sq; 3505 3506 virtnet_tx_pause(vi, sq); 3507 3508 err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf, 3509 virtnet_sq_free_unused_buf_done); 3510 if (err) 3511 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); 3512 3513 virtnet_tx_resume(vi, sq); 3514 3515 return err; 3516} 3517 3518/* 3519 * Send command via the control virtqueue and check status. Commands 3520 * supported by the hypervisor, as indicated by feature bits, should 3521 * never fail unless improperly formatted. 3522 */ 3523static bool virtnet_send_command_reply(struct virtnet_info *vi, u8 class, u8 cmd, 3524 struct scatterlist *out, 3525 struct scatterlist *in) 3526{ 3527 struct scatterlist *sgs[5], hdr, stat; 3528 u32 out_num = 0, tmp, in_num = 0; 3529 bool ok; 3530 int ret; 3531 3532 /* Caller should know better */ 3533 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 3534 3535 mutex_lock(&vi->cvq_lock); 3536 vi->ctrl->status = ~0; 3537 vi->ctrl->hdr.class = class; 3538 vi->ctrl->hdr.cmd = cmd; 3539 /* Add header */ 3540 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 3541 sgs[out_num++] = &hdr; 3542 3543 if (out) 3544 sgs[out_num++] = out; 3545 3546 /* Add return status. */ 3547 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 3548 sgs[out_num + in_num++] = &stat; 3549 3550 if (in) 3551 sgs[out_num + in_num++] = in; 3552 3553 BUG_ON(out_num + in_num > ARRAY_SIZE(sgs)); 3554 ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC); 3555 if (ret < 0) { 3556 dev_warn(&vi->vdev->dev, 3557 "Failed to add sgs for command vq: %d\n.", ret); 3558 mutex_unlock(&vi->cvq_lock); 3559 return false; 3560 } 3561 3562 if (unlikely(!virtqueue_kick(vi->cvq))) 3563 goto unlock; 3564 3565 /* Spin for a response, the kick causes an ioport write, trapping 3566 * into the hypervisor, so the request should be handled immediately. 3567 */ 3568 while (!virtqueue_get_buf(vi->cvq, &tmp) && 3569 !virtqueue_is_broken(vi->cvq)) { 3570 cond_resched(); 3571 cpu_relax(); 3572 } 3573 3574unlock: 3575 ok = vi->ctrl->status == VIRTIO_NET_OK; 3576 mutex_unlock(&vi->cvq_lock); 3577 return ok; 3578} 3579 3580static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 3581 struct scatterlist *out) 3582{ 3583 return virtnet_send_command_reply(vi, class, cmd, out, NULL); 3584} 3585 3586static int virtnet_set_mac_address(struct net_device *dev, void *p) 3587{ 3588 struct virtnet_info *vi = netdev_priv(dev); 3589 struct virtio_device *vdev = vi->vdev; 3590 int ret; 3591 struct sockaddr *addr; 3592 struct scatterlist sg; 3593 3594 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 3595 return -EOPNOTSUPP; 3596 3597 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 3598 if (!addr) 3599 return -ENOMEM; 3600 3601 ret = eth_prepare_mac_addr_change(dev, addr); 3602 if (ret) 3603 goto out; 3604 3605 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 3606 sg_init_one(&sg, addr->sa_data, dev->addr_len); 3607 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 3608 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 3609 dev_warn(&vdev->dev, 3610 "Failed to set mac address by vq command.\n"); 3611 ret = -EINVAL; 3612 goto out; 3613 } 3614 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 3615 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3616 unsigned int i; 3617 3618 /* Naturally, this has an atomicity problem. */ 3619 for (i = 0; i < dev->addr_len; i++) 3620 virtio_cwrite8(vdev, 3621 offsetof(struct virtio_net_config, mac) + 3622 i, addr->sa_data[i]); 3623 } 3624 3625 eth_commit_mac_addr_change(dev, p); 3626 ret = 0; 3627 3628out: 3629 kfree(addr); 3630 return ret; 3631} 3632 3633static void virtnet_stats(struct net_device *dev, 3634 struct rtnl_link_stats64 *tot) 3635{ 3636 struct virtnet_info *vi = netdev_priv(dev); 3637 unsigned int start; 3638 int i; 3639 3640 for (i = 0; i < vi->max_queue_pairs; i++) { 3641 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops; 3642 struct receive_queue *rq = &vi->rq[i]; 3643 struct send_queue *sq = &vi->sq[i]; 3644 3645 do { 3646 start = u64_stats_fetch_begin(&sq->stats.syncp); 3647 tpackets = u64_stats_read(&sq->stats.packets); 3648 tbytes = u64_stats_read(&sq->stats.bytes); 3649 terrors = u64_stats_read(&sq->stats.tx_timeouts); 3650 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 3651 3652 do { 3653 start = u64_stats_fetch_begin(&rq->stats.syncp); 3654 rpackets = u64_stats_read(&rq->stats.packets); 3655 rbytes = u64_stats_read(&rq->stats.bytes); 3656 rdrops = u64_stats_read(&rq->stats.drops); 3657 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 3658 3659 tot->rx_packets += rpackets; 3660 tot->tx_packets += tpackets; 3661 tot->rx_bytes += rbytes; 3662 tot->tx_bytes += tbytes; 3663 tot->rx_dropped += rdrops; 3664 tot->tx_errors += terrors; 3665 } 3666 3667 tot->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 3668 tot->tx_fifo_errors = DEV_STATS_READ(dev, tx_fifo_errors); 3669 tot->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors); 3670 tot->rx_frame_errors = DEV_STATS_READ(dev, rx_frame_errors); 3671} 3672 3673static void virtnet_ack_link_announce(struct virtnet_info *vi) 3674{ 3675 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 3676 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 3677 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 3678} 3679 3680static bool virtnet_commit_rss_command(struct virtnet_info *vi); 3681 3682static void virtnet_rss_update_by_qpairs(struct virtnet_info *vi, u16 queue_pairs) 3683{ 3684 u32 indir_val = 0; 3685 int i = 0; 3686 3687 for (; i < vi->rss_indir_table_size; ++i) { 3688 indir_val = ethtool_rxfh_indir_default(i, queue_pairs); 3689 vi->rss_hdr->indirection_table[i] = cpu_to_le16(indir_val); 3690 } 3691 vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs); 3692} 3693 3694static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 3695{ 3696 struct virtio_net_ctrl_mq *mq __free(kfree) = NULL; 3697 struct virtio_net_rss_config_hdr *old_rss_hdr; 3698 struct virtio_net_rss_config_trailer old_rss_trailer; 3699 struct net_device *dev = vi->dev; 3700 struct scatterlist sg; 3701 3702 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 3703 return 0; 3704 3705 /* Firstly check if we need update rss. Do updating if both (1) rss enabled and 3706 * (2) no user configuration. 3707 * 3708 * During rss command processing, device updates queue_pairs using rss.max_tx_vq. That is, 3709 * the device updates queue_pairs together with rss, so we can skip the separate queue_pairs 3710 * update (VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET below) and return directly. 3711 */ 3712 if (vi->has_rss && !netif_is_rxfh_configured(dev)) { 3713 old_rss_hdr = vi->rss_hdr; 3714 old_rss_trailer = vi->rss_trailer; 3715 vi->rss_hdr = devm_kzalloc(&vi->vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); 3716 if (!vi->rss_hdr) { 3717 vi->rss_hdr = old_rss_hdr; 3718 return -ENOMEM; 3719 } 3720 3721 *vi->rss_hdr = *old_rss_hdr; 3722 virtnet_rss_update_by_qpairs(vi, queue_pairs); 3723 3724 if (!virtnet_commit_rss_command(vi)) { 3725 /* restore ctrl_rss if commit_rss_command failed */ 3726 devm_kfree(&vi->vdev->dev, vi->rss_hdr); 3727 vi->rss_hdr = old_rss_hdr; 3728 vi->rss_trailer = old_rss_trailer; 3729 3730 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d, because committing RSS failed\n", 3731 queue_pairs); 3732 return -EINVAL; 3733 } 3734 devm_kfree(&vi->vdev->dev, old_rss_hdr); 3735 goto succ; 3736 } 3737 3738 mq = kzalloc(sizeof(*mq), GFP_KERNEL); 3739 if (!mq) 3740 return -ENOMEM; 3741 3742 mq->virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 3743 sg_init_one(&sg, mq, sizeof(*mq)); 3744 3745 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 3746 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 3747 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 3748 queue_pairs); 3749 return -EINVAL; 3750 } 3751succ: 3752 vi->curr_queue_pairs = queue_pairs; 3753 if (dev->flags & IFF_UP) { 3754 local_bh_disable(); 3755 for (int i = 0; i < vi->curr_queue_pairs; ++i) 3756 virtqueue_napi_schedule(&vi->rq[i].napi, vi->rq[i].vq); 3757 local_bh_enable(); 3758 } 3759 3760 return 0; 3761} 3762 3763static int virtnet_close(struct net_device *dev) 3764{ 3765 struct virtnet_info *vi = netdev_priv(dev); 3766 int i; 3767 3768 /* Prevent the config change callback from changing carrier 3769 * after close 3770 */ 3771 virtio_config_driver_disable(vi->vdev); 3772 /* Stop getting status/speed updates: we don't care until next 3773 * open 3774 */ 3775 cancel_work_sync(&vi->config_work); 3776 3777 for (i = 0; i < vi->max_queue_pairs; i++) { 3778 virtnet_disable_queue_pair(vi, i); 3779 virtnet_cancel_dim(vi, &vi->rq[i].dim); 3780 } 3781 3782 netif_carrier_off(dev); 3783 3784 return 0; 3785} 3786 3787static void virtnet_rx_mode_work(struct work_struct *work) 3788{ 3789 struct virtnet_info *vi = 3790 container_of(work, struct virtnet_info, rx_mode_work); 3791 u8 *promisc_allmulti __free(kfree) = NULL; 3792 struct net_device *dev = vi->dev; 3793 struct scatterlist sg[2]; 3794 struct virtio_net_ctrl_mac *mac_data; 3795 struct netdev_hw_addr *ha; 3796 int uc_count; 3797 int mc_count; 3798 void *buf; 3799 int i; 3800 3801 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 3802 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 3803 return; 3804 3805 promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL); 3806 if (!promisc_allmulti) { 3807 dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n"); 3808 return; 3809 } 3810 3811 rtnl_lock(); 3812 3813 *promisc_allmulti = !!(dev->flags & IFF_PROMISC); 3814 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); 3815 3816 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 3817 VIRTIO_NET_CTRL_RX_PROMISC, sg)) 3818 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 3819 *promisc_allmulti ? "en" : "dis"); 3820 3821 *promisc_allmulti = !!(dev->flags & IFF_ALLMULTI); 3822 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); 3823 3824 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 3825 VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 3826 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 3827 *promisc_allmulti ? "en" : "dis"); 3828 3829 netif_addr_lock_bh(dev); 3830 3831 uc_count = netdev_uc_count(dev); 3832 mc_count = netdev_mc_count(dev); 3833 /* MAC filter - use one buffer for both lists */ 3834 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 3835 (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 3836 mac_data = buf; 3837 if (!buf) { 3838 netif_addr_unlock_bh(dev); 3839 rtnl_unlock(); 3840 return; 3841 } 3842 3843 sg_init_table(sg, 2); 3844 3845 /* Store the unicast list and count in the front of the buffer */ 3846 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 3847 i = 0; 3848 netdev_for_each_uc_addr(ha, dev) 3849 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 3850 3851 sg_set_buf(&sg[0], mac_data, 3852 sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 3853 3854 /* multicast list and count fill the end */ 3855 mac_data = (void *)&mac_data->macs[uc_count][0]; 3856 3857 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 3858 i = 0; 3859 netdev_for_each_mc_addr(ha, dev) 3860 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 3861 3862 netif_addr_unlock_bh(dev); 3863 3864 sg_set_buf(&sg[1], mac_data, 3865 sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 3866 3867 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 3868 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 3869 dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 3870 3871 rtnl_unlock(); 3872 3873 kfree(buf); 3874} 3875 3876static void virtnet_set_rx_mode(struct net_device *dev) 3877{ 3878 struct virtnet_info *vi = netdev_priv(dev); 3879 3880 if (vi->rx_mode_work_enabled) 3881 schedule_work(&vi->rx_mode_work); 3882} 3883 3884static int virtnet_vlan_rx_add_vid(struct net_device *dev, 3885 __be16 proto, u16 vid) 3886{ 3887 struct virtnet_info *vi = netdev_priv(dev); 3888 __virtio16 *_vid __free(kfree) = NULL; 3889 struct scatterlist sg; 3890 3891 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL); 3892 if (!_vid) 3893 return -ENOMEM; 3894 3895 *_vid = cpu_to_virtio16(vi->vdev, vid); 3896 sg_init_one(&sg, _vid, sizeof(*_vid)); 3897 3898 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 3899 VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 3900 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 3901 return 0; 3902} 3903 3904static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 3905 __be16 proto, u16 vid) 3906{ 3907 struct virtnet_info *vi = netdev_priv(dev); 3908 __virtio16 *_vid __free(kfree) = NULL; 3909 struct scatterlist sg; 3910 3911 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL); 3912 if (!_vid) 3913 return -ENOMEM; 3914 3915 *_vid = cpu_to_virtio16(vi->vdev, vid); 3916 sg_init_one(&sg, _vid, sizeof(*_vid)); 3917 3918 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 3919 VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 3920 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 3921 return 0; 3922} 3923 3924static void virtnet_clean_affinity(struct virtnet_info *vi) 3925{ 3926 int i; 3927 3928 if (vi->affinity_hint_set) { 3929 for (i = 0; i < vi->max_queue_pairs; i++) { 3930 virtqueue_set_affinity(vi->rq[i].vq, NULL); 3931 virtqueue_set_affinity(vi->sq[i].vq, NULL); 3932 } 3933 3934 vi->affinity_hint_set = false; 3935 } 3936} 3937 3938static void virtnet_set_affinity(struct virtnet_info *vi) 3939{ 3940 cpumask_var_t mask; 3941 int stragglers; 3942 int group_size; 3943 int i, start = 0, cpu; 3944 int num_cpu; 3945 int stride; 3946 3947 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 3948 virtnet_clean_affinity(vi); 3949 return; 3950 } 3951 3952 num_cpu = num_online_cpus(); 3953 stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 3954 stragglers = num_cpu >= vi->curr_queue_pairs ? 3955 num_cpu % vi->curr_queue_pairs : 3956 0; 3957 3958 for (i = 0; i < vi->curr_queue_pairs; i++) { 3959 group_size = stride + (i < stragglers ? 1 : 0); 3960 3961 for_each_online_cpu_wrap(cpu, start) { 3962 if (!group_size--) { 3963 start = cpu; 3964 break; 3965 } 3966 cpumask_set_cpu(cpu, mask); 3967 } 3968 3969 virtqueue_set_affinity(vi->rq[i].vq, mask); 3970 virtqueue_set_affinity(vi->sq[i].vq, mask); 3971 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 3972 cpumask_clear(mask); 3973 } 3974 3975 vi->affinity_hint_set = true; 3976 free_cpumask_var(mask); 3977} 3978 3979static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 3980{ 3981 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3982 node); 3983 virtnet_set_affinity(vi); 3984 return 0; 3985} 3986 3987static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 3988{ 3989 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3990 node_dead); 3991 virtnet_set_affinity(vi); 3992 return 0; 3993} 3994 3995static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 3996{ 3997 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3998 node); 3999 4000 virtnet_clean_affinity(vi); 4001 return 0; 4002} 4003 4004static enum cpuhp_state virtionet_online; 4005 4006static int virtnet_cpu_notif_add(struct virtnet_info *vi) 4007{ 4008 int ret; 4009 4010 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 4011 if (ret) 4012 return ret; 4013 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 4014 &vi->node_dead); 4015 if (!ret) 4016 return ret; 4017 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 4018 return ret; 4019} 4020 4021static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 4022{ 4023 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 4024 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 4025 &vi->node_dead); 4026} 4027 4028static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4029 u16 vqn, u32 max_usecs, u32 max_packets) 4030{ 4031 struct virtio_net_ctrl_coal_vq *coal_vq __free(kfree) = NULL; 4032 struct scatterlist sgs; 4033 4034 coal_vq = kzalloc(sizeof(*coal_vq), GFP_KERNEL); 4035 if (!coal_vq) 4036 return -ENOMEM; 4037 4038 coal_vq->vqn = cpu_to_le16(vqn); 4039 coal_vq->coal.max_usecs = cpu_to_le32(max_usecs); 4040 coal_vq->coal.max_packets = cpu_to_le32(max_packets); 4041 sg_init_one(&sgs, coal_vq, sizeof(*coal_vq)); 4042 4043 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 4044 VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET, 4045 &sgs)) 4046 return -EINVAL; 4047 4048 return 0; 4049} 4050 4051static int virtnet_send_rx_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4052 u16 queue, u32 max_usecs, 4053 u32 max_packets) 4054{ 4055 int err; 4056 4057 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 4058 return -EOPNOTSUPP; 4059 4060 err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue), 4061 max_usecs, max_packets); 4062 if (err) 4063 return err; 4064 4065 vi->rq[queue].intr_coal.max_usecs = max_usecs; 4066 vi->rq[queue].intr_coal.max_packets = max_packets; 4067 4068 return 0; 4069} 4070 4071static int virtnet_send_tx_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4072 u16 queue, u32 max_usecs, 4073 u32 max_packets) 4074{ 4075 int err; 4076 4077 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 4078 return -EOPNOTSUPP; 4079 4080 err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue), 4081 max_usecs, max_packets); 4082 if (err) 4083 return err; 4084 4085 vi->sq[queue].intr_coal.max_usecs = max_usecs; 4086 vi->sq[queue].intr_coal.max_packets = max_packets; 4087 4088 return 0; 4089} 4090 4091static void virtnet_get_ringparam(struct net_device *dev, 4092 struct ethtool_ringparam *ring, 4093 struct kernel_ethtool_ringparam *kernel_ring, 4094 struct netlink_ext_ack *extack) 4095{ 4096 struct virtnet_info *vi = netdev_priv(dev); 4097 4098 ring->rx_max_pending = vi->rq[0].vq->num_max; 4099 ring->tx_max_pending = vi->sq[0].vq->num_max; 4100 ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 4101 ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 4102} 4103 4104static int virtnet_set_ringparam(struct net_device *dev, 4105 struct ethtool_ringparam *ring, 4106 struct kernel_ethtool_ringparam *kernel_ring, 4107 struct netlink_ext_ack *extack) 4108{ 4109 struct virtnet_info *vi = netdev_priv(dev); 4110 u32 rx_pending, tx_pending; 4111 struct receive_queue *rq; 4112 struct send_queue *sq; 4113 int i, err; 4114 4115 if (ring->rx_mini_pending || ring->rx_jumbo_pending) 4116 return -EINVAL; 4117 4118 rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 4119 tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 4120 4121 if (ring->rx_pending == rx_pending && 4122 ring->tx_pending == tx_pending) 4123 return 0; 4124 4125 if (ring->rx_pending > vi->rq[0].vq->num_max) 4126 return -EINVAL; 4127 4128 if (ring->tx_pending > vi->sq[0].vq->num_max) 4129 return -EINVAL; 4130 4131 for (i = 0; i < vi->max_queue_pairs; i++) { 4132 rq = vi->rq + i; 4133 sq = vi->sq + i; 4134 4135 if (ring->tx_pending != tx_pending) { 4136 err = virtnet_tx_resize(vi, sq, ring->tx_pending); 4137 if (err) 4138 return err; 4139 4140 /* Upon disabling and re-enabling a transmit virtqueue, the device must 4141 * set the coalescing parameters of the virtqueue to those configured 4142 * through the VIRTIO_NET_CTRL_NOTF_COAL_TX_SET command, or, if the driver 4143 * did not set any TX coalescing parameters, to 0. 4144 */ 4145 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, i, 4146 vi->intr_coal_tx.max_usecs, 4147 vi->intr_coal_tx.max_packets); 4148 4149 /* Don't break the tx resize action if the vq coalescing is not 4150 * supported. The same is true for rx resize below. 4151 */ 4152 if (err && err != -EOPNOTSUPP) 4153 return err; 4154 } 4155 4156 if (ring->rx_pending != rx_pending) { 4157 err = virtnet_rx_resize(vi, rq, ring->rx_pending); 4158 if (err) 4159 return err; 4160 4161 /* The reason is same as the transmit virtqueue reset */ 4162 mutex_lock(&vi->rq[i].dim_lock); 4163 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, i, 4164 vi->intr_coal_rx.max_usecs, 4165 vi->intr_coal_rx.max_packets); 4166 mutex_unlock(&vi->rq[i].dim_lock); 4167 if (err && err != -EOPNOTSUPP) 4168 return err; 4169 } 4170 } 4171 4172 return 0; 4173} 4174 4175static bool virtnet_commit_rss_command(struct virtnet_info *vi) 4176{ 4177 struct net_device *dev = vi->dev; 4178 struct scatterlist sgs[2]; 4179 4180 /* prepare sgs */ 4181 sg_init_table(sgs, 2); 4182 sg_set_buf(&sgs[0], vi->rss_hdr, virtnet_rss_hdr_size(vi)); 4183 sg_set_buf(&sgs[1], &vi->rss_trailer, virtnet_rss_trailer_size(vi)); 4184 4185 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 4186 vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG 4187 : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) 4188 goto err; 4189 4190 return true; 4191 4192err: 4193 dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); 4194 return false; 4195 4196} 4197 4198static void virtnet_init_default_rss(struct virtnet_info *vi) 4199{ 4200 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_supported); 4201 vi->rss_hash_types_saved = vi->rss_hash_types_supported; 4202 vi->rss_hdr->indirection_table_mask = vi->rss_indir_table_size 4203 ? cpu_to_le16(vi->rss_indir_table_size - 1) : 0; 4204 vi->rss_hdr->unclassified_queue = 0; 4205 4206 virtnet_rss_update_by_qpairs(vi, vi->curr_queue_pairs); 4207 4208 vi->rss_trailer.hash_key_length = vi->rss_key_size; 4209 4210 netdev_rss_key_fill(vi->rss_hash_key_data, vi->rss_key_size); 4211} 4212 4213static int virtnet_get_hashflow(struct net_device *dev, 4214 struct ethtool_rxfh_fields *info) 4215{ 4216 struct virtnet_info *vi = netdev_priv(dev); 4217 4218 info->data = 0; 4219 switch (info->flow_type) { 4220 case TCP_V4_FLOW: 4221 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { 4222 info->data = RXH_IP_SRC | RXH_IP_DST | 4223 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4224 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 4225 info->data = RXH_IP_SRC | RXH_IP_DST; 4226 } 4227 break; 4228 case TCP_V6_FLOW: 4229 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { 4230 info->data = RXH_IP_SRC | RXH_IP_DST | 4231 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4232 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 4233 info->data = RXH_IP_SRC | RXH_IP_DST; 4234 } 4235 break; 4236 case UDP_V4_FLOW: 4237 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { 4238 info->data = RXH_IP_SRC | RXH_IP_DST | 4239 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4240 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 4241 info->data = RXH_IP_SRC | RXH_IP_DST; 4242 } 4243 break; 4244 case UDP_V6_FLOW: 4245 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { 4246 info->data = RXH_IP_SRC | RXH_IP_DST | 4247 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4248 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 4249 info->data = RXH_IP_SRC | RXH_IP_DST; 4250 } 4251 break; 4252 case IPV4_FLOW: 4253 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) 4254 info->data = RXH_IP_SRC | RXH_IP_DST; 4255 4256 break; 4257 case IPV6_FLOW: 4258 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) 4259 info->data = RXH_IP_SRC | RXH_IP_DST; 4260 4261 break; 4262 default: 4263 info->data = 0; 4264 break; 4265 } 4266 4267 return 0; 4268} 4269 4270static int virtnet_set_hashflow(struct net_device *dev, 4271 const struct ethtool_rxfh_fields *info, 4272 struct netlink_ext_ack *extack) 4273{ 4274 struct virtnet_info *vi = netdev_priv(dev); 4275 u32 new_hashtypes = vi->rss_hash_types_saved; 4276 bool is_disable = info->data & RXH_DISCARD; 4277 bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3); 4278 4279 /* supports only 'sd', 'sdfn' and 'r' */ 4280 if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable)) 4281 return -EINVAL; 4282 4283 switch (info->flow_type) { 4284 case TCP_V4_FLOW: 4285 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4); 4286 if (!is_disable) 4287 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 4288 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0); 4289 break; 4290 case UDP_V4_FLOW: 4291 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4); 4292 if (!is_disable) 4293 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 4294 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0); 4295 break; 4296 case IPV4_FLOW: 4297 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4; 4298 if (!is_disable) 4299 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4; 4300 break; 4301 case TCP_V6_FLOW: 4302 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6); 4303 if (!is_disable) 4304 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 4305 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0); 4306 break; 4307 case UDP_V6_FLOW: 4308 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6); 4309 if (!is_disable) 4310 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 4311 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0); 4312 break; 4313 case IPV6_FLOW: 4314 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6; 4315 if (!is_disable) 4316 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6; 4317 break; 4318 default: 4319 /* unsupported flow */ 4320 return -EINVAL; 4321 } 4322 4323 /* if unsupported hashtype was set */ 4324 if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) 4325 return -EINVAL; 4326 4327 if (new_hashtypes != vi->rss_hash_types_saved) { 4328 vi->rss_hash_types_saved = new_hashtypes; 4329 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); 4330 if (vi->dev->features & NETIF_F_RXHASH) 4331 if (!virtnet_commit_rss_command(vi)) 4332 return -EINVAL; 4333 } 4334 4335 return 0; 4336} 4337 4338static void virtnet_get_drvinfo(struct net_device *dev, 4339 struct ethtool_drvinfo *info) 4340{ 4341 struct virtnet_info *vi = netdev_priv(dev); 4342 struct virtio_device *vdev = vi->vdev; 4343 4344 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 4345 strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 4346 strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 4347 4348} 4349 4350/* TODO: Eliminate OOO packets during switching */ 4351static int virtnet_set_channels(struct net_device *dev, 4352 struct ethtool_channels *channels) 4353{ 4354 struct virtnet_info *vi = netdev_priv(dev); 4355 u16 queue_pairs = channels->combined_count; 4356 int err; 4357 4358 /* We don't support separate rx/tx channels. 4359 * We don't allow setting 'other' channels. 4360 */ 4361 if (channels->rx_count || channels->tx_count || channels->other_count) 4362 return -EINVAL; 4363 4364 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 4365 return -EINVAL; 4366 4367 /* For now we don't support modifying channels while XDP is loaded 4368 * also when XDP is loaded all RX queues have XDP programs so we only 4369 * need to check a single RX queue. 4370 */ 4371 if (vi->rq[0].xdp_prog) 4372 return -EINVAL; 4373 4374 cpus_read_lock(); 4375 err = virtnet_set_queues(vi, queue_pairs); 4376 if (err) { 4377 cpus_read_unlock(); 4378 goto err; 4379 } 4380 virtnet_set_affinity(vi); 4381 cpus_read_unlock(); 4382 4383 netif_set_real_num_tx_queues(dev, queue_pairs); 4384 netif_set_real_num_rx_queues(dev, queue_pairs); 4385 err: 4386 return err; 4387} 4388 4389static void virtnet_stats_sprintf(u8 **p, const char *fmt, const char *noq_fmt, 4390 int num, int qid, const struct virtnet_stat_desc *desc) 4391{ 4392 int i; 4393 4394 if (qid < 0) { 4395 for (i = 0; i < num; ++i) 4396 ethtool_sprintf(p, noq_fmt, desc[i].desc); 4397 } else { 4398 for (i = 0; i < num; ++i) 4399 ethtool_sprintf(p, fmt, qid, desc[i].desc); 4400 } 4401} 4402 4403/* qid == -1: for rx/tx queue total field */ 4404static void virtnet_get_stats_string(struct virtnet_info *vi, int type, int qid, u8 **data) 4405{ 4406 const struct virtnet_stat_desc *desc; 4407 const char *fmt, *noq_fmt; 4408 u8 *p = *data; 4409 u32 num; 4410 4411 if (type == VIRTNET_Q_TYPE_CQ && qid >= 0) { 4412 noq_fmt = "cq_hw_%s"; 4413 4414 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { 4415 desc = &virtnet_stats_cvq_desc[0]; 4416 num = ARRAY_SIZE(virtnet_stats_cvq_desc); 4417 4418 virtnet_stats_sprintf(&p, NULL, noq_fmt, num, -1, desc); 4419 } 4420 } 4421 4422 if (type == VIRTNET_Q_TYPE_RX) { 4423 fmt = "rx%u_%s"; 4424 noq_fmt = "rx_%s"; 4425 4426 desc = &virtnet_rq_stats_desc[0]; 4427 num = ARRAY_SIZE(virtnet_rq_stats_desc); 4428 4429 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4430 4431 fmt = "rx%u_hw_%s"; 4432 noq_fmt = "rx_hw_%s"; 4433 4434 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4435 desc = &virtnet_stats_rx_basic_desc[0]; 4436 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4437 4438 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4439 } 4440 4441 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4442 desc = &virtnet_stats_rx_csum_desc[0]; 4443 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4444 4445 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4446 } 4447 4448 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4449 desc = &virtnet_stats_rx_speed_desc[0]; 4450 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4451 4452 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4453 } 4454 } 4455 4456 if (type == VIRTNET_Q_TYPE_TX) { 4457 fmt = "tx%u_%s"; 4458 noq_fmt = "tx_%s"; 4459 4460 desc = &virtnet_sq_stats_desc[0]; 4461 num = ARRAY_SIZE(virtnet_sq_stats_desc); 4462 4463 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4464 4465 fmt = "tx%u_hw_%s"; 4466 noq_fmt = "tx_hw_%s"; 4467 4468 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4469 desc = &virtnet_stats_tx_basic_desc[0]; 4470 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4471 4472 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4473 } 4474 4475 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4476 desc = &virtnet_stats_tx_gso_desc[0]; 4477 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4478 4479 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4480 } 4481 4482 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4483 desc = &virtnet_stats_tx_speed_desc[0]; 4484 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4485 4486 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4487 } 4488 } 4489 4490 *data = p; 4491} 4492 4493struct virtnet_stats_ctx { 4494 /* The stats are write to qstats or ethtool -S */ 4495 bool to_qstat; 4496 4497 /* Used to calculate the offset inside the output buffer. */ 4498 u32 desc_num[3]; 4499 4500 /* The actual supported stat types. */ 4501 u64 bitmap[3]; 4502 4503 /* Used to calculate the reply buffer size. */ 4504 u32 size[3]; 4505 4506 /* Record the output buffer. */ 4507 u64 *data; 4508}; 4509 4510static void virtnet_stats_ctx_init(struct virtnet_info *vi, 4511 struct virtnet_stats_ctx *ctx, 4512 u64 *data, bool to_qstat) 4513{ 4514 u32 queue_type; 4515 4516 ctx->data = data; 4517 ctx->to_qstat = to_qstat; 4518 4519 if (to_qstat) { 4520 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc_qstat); 4521 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc_qstat); 4522 4523 queue_type = VIRTNET_Q_TYPE_RX; 4524 4525 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4526 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC; 4527 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat); 4528 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic); 4529 } 4530 4531 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4532 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM; 4533 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat); 4534 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum); 4535 } 4536 4537 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 4538 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_GSO; 4539 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat); 4540 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_gso); 4541 } 4542 4543 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4544 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED; 4545 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat); 4546 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed); 4547 } 4548 4549 queue_type = VIRTNET_Q_TYPE_TX; 4550 4551 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4552 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC; 4553 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat); 4554 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic); 4555 } 4556 4557 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 4558 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_CSUM; 4559 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat); 4560 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_csum); 4561 } 4562 4563 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4564 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO; 4565 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat); 4566 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso); 4567 } 4568 4569 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4570 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED; 4571 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat); 4572 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed); 4573 } 4574 4575 return; 4576 } 4577 4578 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc); 4579 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc); 4580 4581 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { 4582 queue_type = VIRTNET_Q_TYPE_CQ; 4583 4584 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_CVQ; 4585 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_cvq_desc); 4586 ctx->size[queue_type] += sizeof(struct virtio_net_stats_cvq); 4587 } 4588 4589 queue_type = VIRTNET_Q_TYPE_RX; 4590 4591 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4592 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC; 4593 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4594 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic); 4595 } 4596 4597 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4598 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM; 4599 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4600 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum); 4601 } 4602 4603 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4604 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED; 4605 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4606 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed); 4607 } 4608 4609 queue_type = VIRTNET_Q_TYPE_TX; 4610 4611 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4612 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC; 4613 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4614 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic); 4615 } 4616 4617 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4618 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO; 4619 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4620 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso); 4621 } 4622 4623 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4624 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED; 4625 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4626 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed); 4627 } 4628} 4629 4630/* stats_sum_queue - Calculate the sum of the same fields in sq or rq. 4631 * @sum: the position to store the sum values 4632 * @num: field num 4633 * @q_value: the first queue fields 4634 * @q_num: number of the queues 4635 */ 4636static void stats_sum_queue(u64 *sum, u32 num, u64 *q_value, u32 q_num) 4637{ 4638 u32 step = num; 4639 int i, j; 4640 u64 *p; 4641 4642 for (i = 0; i < num; ++i) { 4643 p = sum + i; 4644 *p = 0; 4645 4646 for (j = 0; j < q_num; ++j) 4647 *p += *(q_value + i + j * step); 4648 } 4649} 4650 4651static void virtnet_fill_total_fields(struct virtnet_info *vi, 4652 struct virtnet_stats_ctx *ctx) 4653{ 4654 u64 *data, *first_rx_q, *first_tx_q; 4655 u32 num_cq, num_rx, num_tx; 4656 4657 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ]; 4658 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX]; 4659 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX]; 4660 4661 first_rx_q = ctx->data + num_rx + num_tx + num_cq; 4662 first_tx_q = first_rx_q + vi->curr_queue_pairs * num_rx; 4663 4664 data = ctx->data; 4665 4666 stats_sum_queue(data, num_rx, first_rx_q, vi->curr_queue_pairs); 4667 4668 data = ctx->data + num_rx; 4669 4670 stats_sum_queue(data, num_tx, first_tx_q, vi->curr_queue_pairs); 4671} 4672 4673static void virtnet_fill_stats_qstat(struct virtnet_info *vi, u32 qid, 4674 struct virtnet_stats_ctx *ctx, 4675 const u8 *base, bool drv_stats, u8 reply_type) 4676{ 4677 const struct virtnet_stat_desc *desc; 4678 const u64_stats_t *v_stat; 4679 u64 offset, bitmap; 4680 const __le64 *v; 4681 u32 queue_type; 4682 int i, num; 4683 4684 queue_type = vq_type(vi, qid); 4685 bitmap = ctx->bitmap[queue_type]; 4686 4687 if (drv_stats) { 4688 if (queue_type == VIRTNET_Q_TYPE_RX) { 4689 desc = &virtnet_rq_stats_desc_qstat[0]; 4690 num = ARRAY_SIZE(virtnet_rq_stats_desc_qstat); 4691 } else { 4692 desc = &virtnet_sq_stats_desc_qstat[0]; 4693 num = ARRAY_SIZE(virtnet_sq_stats_desc_qstat); 4694 } 4695 4696 for (i = 0; i < num; ++i) { 4697 offset = desc[i].qstat_offset / sizeof(*ctx->data); 4698 v_stat = (const u64_stats_t *)(base + desc[i].offset); 4699 ctx->data[offset] = u64_stats_read(v_stat); 4700 } 4701 return; 4702 } 4703 4704 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4705 desc = &virtnet_stats_rx_basic_desc_qstat[0]; 4706 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat); 4707 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC) 4708 goto found; 4709 } 4710 4711 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4712 desc = &virtnet_stats_rx_csum_desc_qstat[0]; 4713 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat); 4714 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM) 4715 goto found; 4716 } 4717 4718 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 4719 desc = &virtnet_stats_rx_gso_desc_qstat[0]; 4720 num = ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat); 4721 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_GSO) 4722 goto found; 4723 } 4724 4725 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4726 desc = &virtnet_stats_rx_speed_desc_qstat[0]; 4727 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat); 4728 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED) 4729 goto found; 4730 } 4731 4732 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4733 desc = &virtnet_stats_tx_basic_desc_qstat[0]; 4734 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat); 4735 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC) 4736 goto found; 4737 } 4738 4739 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 4740 desc = &virtnet_stats_tx_csum_desc_qstat[0]; 4741 num = ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat); 4742 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_CSUM) 4743 goto found; 4744 } 4745 4746 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4747 desc = &virtnet_stats_tx_gso_desc_qstat[0]; 4748 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat); 4749 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO) 4750 goto found; 4751 } 4752 4753 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4754 desc = &virtnet_stats_tx_speed_desc_qstat[0]; 4755 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat); 4756 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED) 4757 goto found; 4758 } 4759 4760 return; 4761 4762found: 4763 for (i = 0; i < num; ++i) { 4764 offset = desc[i].qstat_offset / sizeof(*ctx->data); 4765 v = (const __le64 *)(base + desc[i].offset); 4766 ctx->data[offset] = le64_to_cpu(*v); 4767 } 4768} 4769 4770/* virtnet_fill_stats - copy the stats to qstats or ethtool -S 4771 * The stats source is the device or the driver. 4772 * 4773 * @vi: virtio net info 4774 * @qid: the vq id 4775 * @ctx: stats ctx (initiated by virtnet_stats_ctx_init()) 4776 * @base: pointer to the device reply or the driver stats structure. 4777 * @drv_stats: designate the base type (device reply, driver stats) 4778 * @type: the type of the device reply (if drv_stats is true, this must be zero) 4779 */ 4780static void virtnet_fill_stats(struct virtnet_info *vi, u32 qid, 4781 struct virtnet_stats_ctx *ctx, 4782 const u8 *base, bool drv_stats, u8 reply_type) 4783{ 4784 u32 queue_type, num_rx, num_tx, num_cq; 4785 const struct virtnet_stat_desc *desc; 4786 const u64_stats_t *v_stat; 4787 u64 offset, bitmap; 4788 const __le64 *v; 4789 int i, num; 4790 4791 if (ctx->to_qstat) 4792 return virtnet_fill_stats_qstat(vi, qid, ctx, base, drv_stats, reply_type); 4793 4794 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ]; 4795 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX]; 4796 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX]; 4797 4798 queue_type = vq_type(vi, qid); 4799 bitmap = ctx->bitmap[queue_type]; 4800 4801 /* skip the total fields of pairs */ 4802 offset = num_rx + num_tx; 4803 4804 if (queue_type == VIRTNET_Q_TYPE_TX) { 4805 offset += num_cq + num_rx * vi->curr_queue_pairs + num_tx * (qid / 2); 4806 4807 num = ARRAY_SIZE(virtnet_sq_stats_desc); 4808 if (drv_stats) { 4809 desc = &virtnet_sq_stats_desc[0]; 4810 goto drv_stats; 4811 } 4812 4813 offset += num; 4814 4815 } else if (queue_type == VIRTNET_Q_TYPE_RX) { 4816 offset += num_cq + num_rx * (qid / 2); 4817 4818 num = ARRAY_SIZE(virtnet_rq_stats_desc); 4819 if (drv_stats) { 4820 desc = &virtnet_rq_stats_desc[0]; 4821 goto drv_stats; 4822 } 4823 4824 offset += num; 4825 } 4826 4827 if (bitmap & VIRTIO_NET_STATS_TYPE_CVQ) { 4828 desc = &virtnet_stats_cvq_desc[0]; 4829 num = ARRAY_SIZE(virtnet_stats_cvq_desc); 4830 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_CVQ) 4831 goto found; 4832 4833 offset += num; 4834 } 4835 4836 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4837 desc = &virtnet_stats_rx_basic_desc[0]; 4838 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4839 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC) 4840 goto found; 4841 4842 offset += num; 4843 } 4844 4845 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4846 desc = &virtnet_stats_rx_csum_desc[0]; 4847 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4848 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM) 4849 goto found; 4850 4851 offset += num; 4852 } 4853 4854 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4855 desc = &virtnet_stats_rx_speed_desc[0]; 4856 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4857 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED) 4858 goto found; 4859 4860 offset += num; 4861 } 4862 4863 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4864 desc = &virtnet_stats_tx_basic_desc[0]; 4865 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4866 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC) 4867 goto found; 4868 4869 offset += num; 4870 } 4871 4872 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4873 desc = &virtnet_stats_tx_gso_desc[0]; 4874 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4875 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO) 4876 goto found; 4877 4878 offset += num; 4879 } 4880 4881 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4882 desc = &virtnet_stats_tx_speed_desc[0]; 4883 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4884 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED) 4885 goto found; 4886 4887 offset += num; 4888 } 4889 4890 return; 4891 4892found: 4893 for (i = 0; i < num; ++i) { 4894 v = (const __le64 *)(base + desc[i].offset); 4895 ctx->data[offset + i] = le64_to_cpu(*v); 4896 } 4897 4898 return; 4899 4900drv_stats: 4901 for (i = 0; i < num; ++i) { 4902 v_stat = (const u64_stats_t *)(base + desc[i].offset); 4903 ctx->data[offset + i] = u64_stats_read(v_stat); 4904 } 4905} 4906 4907static int __virtnet_get_hw_stats(struct virtnet_info *vi, 4908 struct virtnet_stats_ctx *ctx, 4909 struct virtio_net_ctrl_queue_stats *req, 4910 int req_size, void *reply, int res_size) 4911{ 4912 struct virtio_net_stats_reply_hdr *hdr; 4913 struct scatterlist sgs_in, sgs_out; 4914 void *p; 4915 u32 qid; 4916 int ok; 4917 4918 sg_init_one(&sgs_out, req, req_size); 4919 sg_init_one(&sgs_in, reply, res_size); 4920 4921 ok = virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, 4922 VIRTIO_NET_CTRL_STATS_GET, 4923 &sgs_out, &sgs_in); 4924 4925 if (!ok) 4926 return ok; 4927 4928 for (p = reply; p - reply < res_size; p += le16_to_cpu(hdr->size)) { 4929 hdr = p; 4930 qid = le16_to_cpu(hdr->vq_index); 4931 virtnet_fill_stats(vi, qid, ctx, p, false, hdr->type); 4932 } 4933 4934 return 0; 4935} 4936 4937static void virtnet_make_stat_req(struct virtnet_info *vi, 4938 struct virtnet_stats_ctx *ctx, 4939 struct virtio_net_ctrl_queue_stats *req, 4940 int qid, int *idx) 4941{ 4942 int qtype = vq_type(vi, qid); 4943 u64 bitmap = ctx->bitmap[qtype]; 4944 4945 if (!bitmap) 4946 return; 4947 4948 req->stats[*idx].vq_index = cpu_to_le16(qid); 4949 req->stats[*idx].types_bitmap[0] = cpu_to_le64(bitmap); 4950 *idx += 1; 4951} 4952 4953/* qid: -1: get stats of all vq. 4954 * > 0: get the stats for the special vq. This must not be cvq. 4955 */ 4956static int virtnet_get_hw_stats(struct virtnet_info *vi, 4957 struct virtnet_stats_ctx *ctx, int qid) 4958{ 4959 int qnum, i, j, res_size, qtype, last_vq, first_vq; 4960 struct virtio_net_ctrl_queue_stats *req; 4961 bool enable_cvq; 4962 void *reply; 4963 int ok; 4964 4965 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) 4966 return 0; 4967 4968 if (qid == -1) { 4969 last_vq = vi->curr_queue_pairs * 2 - 1; 4970 first_vq = 0; 4971 enable_cvq = true; 4972 } else { 4973 last_vq = qid; 4974 first_vq = qid; 4975 enable_cvq = false; 4976 } 4977 4978 qnum = 0; 4979 res_size = 0; 4980 for (i = first_vq; i <= last_vq ; ++i) { 4981 qtype = vq_type(vi, i); 4982 if (ctx->bitmap[qtype]) { 4983 ++qnum; 4984 res_size += ctx->size[qtype]; 4985 } 4986 } 4987 4988 if (enable_cvq && ctx->bitmap[VIRTNET_Q_TYPE_CQ]) { 4989 res_size += ctx->size[VIRTNET_Q_TYPE_CQ]; 4990 qnum += 1; 4991 } 4992 4993 req = kcalloc(qnum, sizeof(*req), GFP_KERNEL); 4994 if (!req) 4995 return -ENOMEM; 4996 4997 reply = kmalloc(res_size, GFP_KERNEL); 4998 if (!reply) { 4999 kfree(req); 5000 return -ENOMEM; 5001 } 5002 5003 j = 0; 5004 for (i = first_vq; i <= last_vq ; ++i) 5005 virtnet_make_stat_req(vi, ctx, req, i, &j); 5006 5007 if (enable_cvq) 5008 virtnet_make_stat_req(vi, ctx, req, vi->max_queue_pairs * 2, &j); 5009 5010 ok = __virtnet_get_hw_stats(vi, ctx, req, sizeof(*req) * j, reply, res_size); 5011 5012 kfree(req); 5013 kfree(reply); 5014 5015 return ok; 5016} 5017 5018static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 5019{ 5020 struct virtnet_info *vi = netdev_priv(dev); 5021 unsigned int i; 5022 u8 *p = data; 5023 5024 switch (stringset) { 5025 case ETH_SS_STATS: 5026 /* Generate the total field names. */ 5027 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, -1, &p); 5028 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, -1, &p); 5029 5030 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_CQ, 0, &p); 5031 5032 for (i = 0; i < vi->curr_queue_pairs; ++i) 5033 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, i, &p); 5034 5035 for (i = 0; i < vi->curr_queue_pairs; ++i) 5036 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, i, &p); 5037 break; 5038 } 5039} 5040 5041static int virtnet_get_sset_count(struct net_device *dev, int sset) 5042{ 5043 struct virtnet_info *vi = netdev_priv(dev); 5044 struct virtnet_stats_ctx ctx = {0}; 5045 u32 pair_count; 5046 5047 switch (sset) { 5048 case ETH_SS_STATS: 5049 virtnet_stats_ctx_init(vi, &ctx, NULL, false); 5050 5051 pair_count = ctx.desc_num[VIRTNET_Q_TYPE_RX] + ctx.desc_num[VIRTNET_Q_TYPE_TX]; 5052 5053 return pair_count + ctx.desc_num[VIRTNET_Q_TYPE_CQ] + 5054 vi->curr_queue_pairs * pair_count; 5055 default: 5056 return -EOPNOTSUPP; 5057 } 5058} 5059 5060static void virtnet_get_ethtool_stats(struct net_device *dev, 5061 struct ethtool_stats *stats, u64 *data) 5062{ 5063 struct virtnet_info *vi = netdev_priv(dev); 5064 struct virtnet_stats_ctx ctx = {0}; 5065 unsigned int start, i; 5066 const u8 *stats_base; 5067 5068 virtnet_stats_ctx_init(vi, &ctx, data, false); 5069 if (virtnet_get_hw_stats(vi, &ctx, -1)) 5070 dev_warn(&vi->dev->dev, "Failed to get hw stats.\n"); 5071 5072 for (i = 0; i < vi->curr_queue_pairs; i++) { 5073 struct receive_queue *rq = &vi->rq[i]; 5074 struct send_queue *sq = &vi->sq[i]; 5075 5076 stats_base = (const u8 *)&rq->stats; 5077 do { 5078 start = u64_stats_fetch_begin(&rq->stats.syncp); 5079 virtnet_fill_stats(vi, i * 2, &ctx, stats_base, true, 0); 5080 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 5081 5082 stats_base = (const u8 *)&sq->stats; 5083 do { 5084 start = u64_stats_fetch_begin(&sq->stats.syncp); 5085 virtnet_fill_stats(vi, i * 2 + 1, &ctx, stats_base, true, 0); 5086 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 5087 } 5088 5089 virtnet_fill_total_fields(vi, &ctx); 5090} 5091 5092static void virtnet_get_channels(struct net_device *dev, 5093 struct ethtool_channels *channels) 5094{ 5095 struct virtnet_info *vi = netdev_priv(dev); 5096 5097 channels->combined_count = vi->curr_queue_pairs; 5098 channels->max_combined = vi->max_queue_pairs; 5099 channels->max_other = 0; 5100 channels->rx_count = 0; 5101 channels->tx_count = 0; 5102 channels->other_count = 0; 5103} 5104 5105static int virtnet_set_link_ksettings(struct net_device *dev, 5106 const struct ethtool_link_ksettings *cmd) 5107{ 5108 struct virtnet_info *vi = netdev_priv(dev); 5109 5110 return ethtool_virtdev_set_link_ksettings(dev, cmd, 5111 &vi->speed, &vi->duplex); 5112} 5113 5114static int virtnet_get_link_ksettings(struct net_device *dev, 5115 struct ethtool_link_ksettings *cmd) 5116{ 5117 struct virtnet_info *vi = netdev_priv(dev); 5118 5119 cmd->base.speed = vi->speed; 5120 cmd->base.duplex = vi->duplex; 5121 cmd->base.port = PORT_OTHER; 5122 5123 return 0; 5124} 5125 5126static int virtnet_send_tx_notf_coal_cmds(struct virtnet_info *vi, 5127 struct ethtool_coalesce *ec) 5128{ 5129 struct virtio_net_ctrl_coal_tx *coal_tx __free(kfree) = NULL; 5130 struct scatterlist sgs_tx; 5131 int i; 5132 5133 coal_tx = kzalloc(sizeof(*coal_tx), GFP_KERNEL); 5134 if (!coal_tx) 5135 return -ENOMEM; 5136 5137 coal_tx->tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); 5138 coal_tx->tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); 5139 sg_init_one(&sgs_tx, coal_tx, sizeof(*coal_tx)); 5140 5141 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 5142 VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, 5143 &sgs_tx)) 5144 return -EINVAL; 5145 5146 vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs; 5147 vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames; 5148 for (i = 0; i < vi->max_queue_pairs; i++) { 5149 vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs; 5150 vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames; 5151 } 5152 5153 return 0; 5154} 5155 5156static int virtnet_send_rx_notf_coal_cmds(struct virtnet_info *vi, 5157 struct ethtool_coalesce *ec) 5158{ 5159 struct virtio_net_ctrl_coal_rx *coal_rx __free(kfree) = NULL; 5160 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce; 5161 struct scatterlist sgs_rx; 5162 int i; 5163 5164 if (rx_ctrl_dim_on && !virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 5165 return -EOPNOTSUPP; 5166 5167 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != vi->intr_coal_rx.max_usecs || 5168 ec->rx_max_coalesced_frames != vi->intr_coal_rx.max_packets)) 5169 return -EINVAL; 5170 5171 if (rx_ctrl_dim_on && !vi->rx_dim_enabled) { 5172 vi->rx_dim_enabled = true; 5173 for (i = 0; i < vi->max_queue_pairs; i++) { 5174 mutex_lock(&vi->rq[i].dim_lock); 5175 vi->rq[i].dim_enabled = true; 5176 mutex_unlock(&vi->rq[i].dim_lock); 5177 } 5178 return 0; 5179 } 5180 5181 coal_rx = kzalloc(sizeof(*coal_rx), GFP_KERNEL); 5182 if (!coal_rx) 5183 return -ENOMEM; 5184 5185 if (!rx_ctrl_dim_on && vi->rx_dim_enabled) { 5186 vi->rx_dim_enabled = false; 5187 for (i = 0; i < vi->max_queue_pairs; i++) { 5188 mutex_lock(&vi->rq[i].dim_lock); 5189 vi->rq[i].dim_enabled = false; 5190 mutex_unlock(&vi->rq[i].dim_lock); 5191 } 5192 } 5193 5194 /* Since the per-queue coalescing params can be set, 5195 * we need apply the global new params even if they 5196 * are not updated. 5197 */ 5198 coal_rx->rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); 5199 coal_rx->rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); 5200 sg_init_one(&sgs_rx, coal_rx, sizeof(*coal_rx)); 5201 5202 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 5203 VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, 5204 &sgs_rx)) 5205 return -EINVAL; 5206 5207 vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs; 5208 vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames; 5209 for (i = 0; i < vi->max_queue_pairs; i++) { 5210 mutex_lock(&vi->rq[i].dim_lock); 5211 vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs; 5212 vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames; 5213 mutex_unlock(&vi->rq[i].dim_lock); 5214 } 5215 5216 return 0; 5217} 5218 5219static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, 5220 struct ethtool_coalesce *ec) 5221{ 5222 int err; 5223 5224 err = virtnet_send_tx_notf_coal_cmds(vi, ec); 5225 if (err) 5226 return err; 5227 5228 err = virtnet_send_rx_notf_coal_cmds(vi, ec); 5229 if (err) 5230 return err; 5231 5232 return 0; 5233} 5234 5235static int virtnet_send_rx_notf_coal_vq_cmds(struct virtnet_info *vi, 5236 struct ethtool_coalesce *ec, 5237 u16 queue) 5238{ 5239 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce; 5240 u32 max_usecs, max_packets; 5241 bool cur_rx_dim; 5242 int err; 5243 5244 mutex_lock(&vi->rq[queue].dim_lock); 5245 cur_rx_dim = vi->rq[queue].dim_enabled; 5246 max_usecs = vi->rq[queue].intr_coal.max_usecs; 5247 max_packets = vi->rq[queue].intr_coal.max_packets; 5248 5249 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != max_usecs || 5250 ec->rx_max_coalesced_frames != max_packets)) { 5251 mutex_unlock(&vi->rq[queue].dim_lock); 5252 return -EINVAL; 5253 } 5254 5255 if (rx_ctrl_dim_on && !cur_rx_dim) { 5256 vi->rq[queue].dim_enabled = true; 5257 mutex_unlock(&vi->rq[queue].dim_lock); 5258 return 0; 5259 } 5260 5261 if (!rx_ctrl_dim_on && cur_rx_dim) 5262 vi->rq[queue].dim_enabled = false; 5263 5264 /* If no params are updated, userspace ethtool will 5265 * reject the modification. 5266 */ 5267 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, queue, 5268 ec->rx_coalesce_usecs, 5269 ec->rx_max_coalesced_frames); 5270 mutex_unlock(&vi->rq[queue].dim_lock); 5271 return err; 5272} 5273 5274static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, 5275 struct ethtool_coalesce *ec, 5276 u16 queue) 5277{ 5278 int err; 5279 5280 err = virtnet_send_rx_notf_coal_vq_cmds(vi, ec, queue); 5281 if (err) 5282 return err; 5283 5284 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, queue, 5285 ec->tx_coalesce_usecs, 5286 ec->tx_max_coalesced_frames); 5287 if (err) 5288 return err; 5289 5290 return 0; 5291} 5292 5293static void virtnet_rx_dim_work(struct work_struct *work) 5294{ 5295 struct dim *dim = container_of(work, struct dim, work); 5296 struct receive_queue *rq = container_of(dim, 5297 struct receive_queue, dim); 5298 struct virtnet_info *vi = rq->vq->vdev->priv; 5299 struct net_device *dev = vi->dev; 5300 struct dim_cq_moder update_moder; 5301 int qnum, err; 5302 5303 qnum = rq - vi->rq; 5304 5305 mutex_lock(&rq->dim_lock); 5306 if (!rq->dim_enabled) 5307 goto out; 5308 5309 update_moder = net_dim_get_rx_irq_moder(dev, dim); 5310 if (update_moder.usec != rq->intr_coal.max_usecs || 5311 update_moder.pkts != rq->intr_coal.max_packets) { 5312 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum, 5313 update_moder.usec, 5314 update_moder.pkts); 5315 if (err) 5316 pr_debug("%s: Failed to send dim parameters on rxq%d\n", 5317 dev->name, qnum); 5318 } 5319out: 5320 dim->state = DIM_START_MEASURE; 5321 mutex_unlock(&rq->dim_lock); 5322} 5323 5324static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) 5325{ 5326 /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL 5327 * or VIRTIO_NET_F_VQ_NOTF_COAL feature is negotiated. 5328 */ 5329 if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) 5330 return -EOPNOTSUPP; 5331 5332 if (ec->tx_max_coalesced_frames > 1 || 5333 ec->rx_max_coalesced_frames != 1) 5334 return -EINVAL; 5335 5336 return 0; 5337} 5338 5339static int virtnet_should_update_vq_weight(int dev_flags, int weight, 5340 int vq_weight, bool *should_update) 5341{ 5342 if (weight ^ vq_weight) { 5343 if (dev_flags & IFF_UP) 5344 return -EBUSY; 5345 *should_update = true; 5346 } 5347 5348 return 0; 5349} 5350 5351static int virtnet_set_coalesce(struct net_device *dev, 5352 struct ethtool_coalesce *ec, 5353 struct kernel_ethtool_coalesce *kernel_coal, 5354 struct netlink_ext_ack *extack) 5355{ 5356 struct virtnet_info *vi = netdev_priv(dev); 5357 int ret, queue_number, napi_weight, i; 5358 bool update_napi = false; 5359 5360 /* Can't change NAPI weight if the link is up */ 5361 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 5362 for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) { 5363 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 5364 vi->sq[queue_number].napi.weight, 5365 &update_napi); 5366 if (ret) 5367 return ret; 5368 5369 if (update_napi) { 5370 /* All queues that belong to [queue_number, vi->max_queue_pairs] will be 5371 * updated for the sake of simplicity, which might not be necessary 5372 */ 5373 break; 5374 } 5375 } 5376 5377 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) 5378 ret = virtnet_send_notf_coal_cmds(vi, ec); 5379 else 5380 ret = virtnet_coal_params_supported(ec); 5381 5382 if (ret) 5383 return ret; 5384 5385 if (update_napi) { 5386 /* xsk xmit depends on the tx napi. So if xsk is active, 5387 * prevent modifications to tx napi. 5388 */ 5389 for (i = queue_number; i < vi->max_queue_pairs; i++) { 5390 if (vi->sq[i].xsk_pool) 5391 return -EBUSY; 5392 } 5393 5394 for (; queue_number < vi->max_queue_pairs; queue_number++) 5395 vi->sq[queue_number].napi.weight = napi_weight; 5396 } 5397 5398 return ret; 5399} 5400 5401static int virtnet_get_coalesce(struct net_device *dev, 5402 struct ethtool_coalesce *ec, 5403 struct kernel_ethtool_coalesce *kernel_coal, 5404 struct netlink_ext_ack *extack) 5405{ 5406 struct virtnet_info *vi = netdev_priv(dev); 5407 5408 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 5409 ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs; 5410 ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs; 5411 ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets; 5412 ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets; 5413 ec->use_adaptive_rx_coalesce = vi->rx_dim_enabled; 5414 } else { 5415 ec->rx_max_coalesced_frames = 1; 5416 5417 if (vi->sq[0].napi.weight) 5418 ec->tx_max_coalesced_frames = 1; 5419 } 5420 5421 return 0; 5422} 5423 5424static int virtnet_set_per_queue_coalesce(struct net_device *dev, 5425 u32 queue, 5426 struct ethtool_coalesce *ec) 5427{ 5428 struct virtnet_info *vi = netdev_priv(dev); 5429 int ret, napi_weight; 5430 bool update_napi = false; 5431 5432 if (queue >= vi->max_queue_pairs) 5433 return -EINVAL; 5434 5435 /* Can't change NAPI weight if the link is up */ 5436 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 5437 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 5438 vi->sq[queue].napi.weight, 5439 &update_napi); 5440 if (ret) 5441 return ret; 5442 5443 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 5444 ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue); 5445 else 5446 ret = virtnet_coal_params_supported(ec); 5447 5448 if (ret) 5449 return ret; 5450 5451 if (update_napi) 5452 vi->sq[queue].napi.weight = napi_weight; 5453 5454 return 0; 5455} 5456 5457static int virtnet_get_per_queue_coalesce(struct net_device *dev, 5458 u32 queue, 5459 struct ethtool_coalesce *ec) 5460{ 5461 struct virtnet_info *vi = netdev_priv(dev); 5462 5463 if (queue >= vi->max_queue_pairs) 5464 return -EINVAL; 5465 5466 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { 5467 mutex_lock(&vi->rq[queue].dim_lock); 5468 ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs; 5469 ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs; 5470 ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets; 5471 ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets; 5472 ec->use_adaptive_rx_coalesce = vi->rq[queue].dim_enabled; 5473 mutex_unlock(&vi->rq[queue].dim_lock); 5474 } else { 5475 ec->rx_max_coalesced_frames = 1; 5476 5477 if (vi->sq[queue].napi.weight) 5478 ec->tx_max_coalesced_frames = 1; 5479 } 5480 5481 return 0; 5482} 5483 5484static void virtnet_init_settings(struct net_device *dev) 5485{ 5486 struct virtnet_info *vi = netdev_priv(dev); 5487 5488 vi->speed = SPEED_UNKNOWN; 5489 vi->duplex = DUPLEX_UNKNOWN; 5490} 5491 5492static u32 virtnet_get_rxfh_key_size(struct net_device *dev) 5493{ 5494 return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; 5495} 5496 5497static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) 5498{ 5499 return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; 5500} 5501 5502static int virtnet_get_rxfh(struct net_device *dev, 5503 struct ethtool_rxfh_param *rxfh) 5504{ 5505 struct virtnet_info *vi = netdev_priv(dev); 5506 int i; 5507 5508 if (rxfh->indir) { 5509 for (i = 0; i < vi->rss_indir_table_size; ++i) 5510 rxfh->indir[i] = le16_to_cpu(vi->rss_hdr->indirection_table[i]); 5511 } 5512 5513 if (rxfh->key) 5514 memcpy(rxfh->key, vi->rss_hash_key_data, vi->rss_key_size); 5515 5516 rxfh->hfunc = ETH_RSS_HASH_TOP; 5517 5518 return 0; 5519} 5520 5521static int virtnet_set_rxfh(struct net_device *dev, 5522 struct ethtool_rxfh_param *rxfh, 5523 struct netlink_ext_ack *extack) 5524{ 5525 struct virtnet_info *vi = netdev_priv(dev); 5526 bool update = false; 5527 int i; 5528 5529 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && 5530 rxfh->hfunc != ETH_RSS_HASH_TOP) 5531 return -EOPNOTSUPP; 5532 5533 if (rxfh->indir) { 5534 if (!vi->has_rss) 5535 return -EOPNOTSUPP; 5536 5537 for (i = 0; i < vi->rss_indir_table_size; ++i) 5538 vi->rss_hdr->indirection_table[i] = cpu_to_le16(rxfh->indir[i]); 5539 update = true; 5540 } 5541 5542 if (rxfh->key) { 5543 /* If either _F_HASH_REPORT or _F_RSS are negotiated, the 5544 * device provides hash calculation capabilities, that is, 5545 * hash_key is configured. 5546 */ 5547 if (!vi->has_rss && !vi->has_rss_hash_report) 5548 return -EOPNOTSUPP; 5549 5550 memcpy(vi->rss_hash_key_data, rxfh->key, vi->rss_key_size); 5551 update = true; 5552 } 5553 5554 if (update) 5555 virtnet_commit_rss_command(vi); 5556 5557 return 0; 5558} 5559 5560static u32 virtnet_get_rx_ring_count(struct net_device *dev) 5561{ 5562 struct virtnet_info *vi = netdev_priv(dev); 5563 5564 return vi->curr_queue_pairs; 5565} 5566 5567static const struct ethtool_ops virtnet_ethtool_ops = { 5568 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 5569 ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, 5570 .get_drvinfo = virtnet_get_drvinfo, 5571 .get_link = ethtool_op_get_link, 5572 .get_ringparam = virtnet_get_ringparam, 5573 .set_ringparam = virtnet_set_ringparam, 5574 .get_strings = virtnet_get_strings, 5575 .get_sset_count = virtnet_get_sset_count, 5576 .get_ethtool_stats = virtnet_get_ethtool_stats, 5577 .set_channels = virtnet_set_channels, 5578 .get_channels = virtnet_get_channels, 5579 .get_ts_info = ethtool_op_get_ts_info, 5580 .get_link_ksettings = virtnet_get_link_ksettings, 5581 .set_link_ksettings = virtnet_set_link_ksettings, 5582 .set_coalesce = virtnet_set_coalesce, 5583 .get_coalesce = virtnet_get_coalesce, 5584 .set_per_queue_coalesce = virtnet_set_per_queue_coalesce, 5585 .get_per_queue_coalesce = virtnet_get_per_queue_coalesce, 5586 .get_rxfh_key_size = virtnet_get_rxfh_key_size, 5587 .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, 5588 .get_rxfh = virtnet_get_rxfh, 5589 .set_rxfh = virtnet_set_rxfh, 5590 .get_rxfh_fields = virtnet_get_hashflow, 5591 .set_rxfh_fields = virtnet_set_hashflow, 5592 .get_rx_ring_count = virtnet_get_rx_ring_count, 5593}; 5594 5595static void virtnet_get_queue_stats_rx(struct net_device *dev, int i, 5596 struct netdev_queue_stats_rx *stats) 5597{ 5598 struct virtnet_info *vi = netdev_priv(dev); 5599 struct receive_queue *rq = &vi->rq[i]; 5600 struct virtnet_stats_ctx ctx = {0}; 5601 5602 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); 5603 5604 virtnet_get_hw_stats(vi, &ctx, i * 2); 5605 virtnet_fill_stats(vi, i * 2, &ctx, (void *)&rq->stats, true, 0); 5606} 5607 5608static void virtnet_get_queue_stats_tx(struct net_device *dev, int i, 5609 struct netdev_queue_stats_tx *stats) 5610{ 5611 struct virtnet_info *vi = netdev_priv(dev); 5612 struct send_queue *sq = &vi->sq[i]; 5613 struct virtnet_stats_ctx ctx = {0}; 5614 5615 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); 5616 5617 virtnet_get_hw_stats(vi, &ctx, i * 2 + 1); 5618 virtnet_fill_stats(vi, i * 2 + 1, &ctx, (void *)&sq->stats, true, 0); 5619} 5620 5621static void virtnet_get_base_stats(struct net_device *dev, 5622 struct netdev_queue_stats_rx *rx, 5623 struct netdev_queue_stats_tx *tx) 5624{ 5625 struct virtnet_info *vi = netdev_priv(dev); 5626 5627 /* The queue stats of the virtio-net will not be reset. So here we 5628 * return 0. 5629 */ 5630 rx->bytes = 0; 5631 rx->packets = 0; 5632 5633 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 5634 rx->hw_drops = 0; 5635 rx->hw_drop_overruns = 0; 5636 } 5637 5638 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 5639 rx->csum_unnecessary = 0; 5640 rx->csum_none = 0; 5641 rx->csum_bad = 0; 5642 } 5643 5644 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 5645 rx->hw_gro_packets = 0; 5646 rx->hw_gro_bytes = 0; 5647 rx->hw_gro_wire_packets = 0; 5648 rx->hw_gro_wire_bytes = 0; 5649 } 5650 5651 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) 5652 rx->hw_drop_ratelimits = 0; 5653 5654 tx->bytes = 0; 5655 tx->packets = 0; 5656 tx->stop = 0; 5657 tx->wake = 0; 5658 5659 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 5660 tx->hw_drops = 0; 5661 tx->hw_drop_errors = 0; 5662 } 5663 5664 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 5665 tx->csum_none = 0; 5666 tx->needs_csum = 0; 5667 } 5668 5669 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 5670 tx->hw_gso_packets = 0; 5671 tx->hw_gso_bytes = 0; 5672 tx->hw_gso_wire_packets = 0; 5673 tx->hw_gso_wire_bytes = 0; 5674 } 5675 5676 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) 5677 tx->hw_drop_ratelimits = 0; 5678 5679 netdev_stat_queue_sum(dev, 5680 dev->real_num_rx_queues, vi->max_queue_pairs, rx, 5681 dev->real_num_tx_queues, vi->max_queue_pairs, tx); 5682} 5683 5684static const struct netdev_stat_ops virtnet_stat_ops = { 5685 .get_queue_stats_rx = virtnet_get_queue_stats_rx, 5686 .get_queue_stats_tx = virtnet_get_queue_stats_tx, 5687 .get_base_stats = virtnet_get_base_stats, 5688}; 5689 5690static void virtnet_freeze_down(struct virtio_device *vdev) 5691{ 5692 struct virtnet_info *vi = vdev->priv; 5693 5694 /* Make sure no work handler is accessing the device */ 5695 flush_work(&vi->config_work); 5696 disable_rx_mode_work(vi); 5697 flush_work(&vi->rx_mode_work); 5698 5699 if (netif_running(vi->dev)) { 5700 rtnl_lock(); 5701 virtnet_close(vi->dev); 5702 rtnl_unlock(); 5703 } 5704 5705 netif_tx_lock_bh(vi->dev); 5706 netif_device_detach(vi->dev); 5707 netif_tx_unlock_bh(vi->dev); 5708} 5709 5710static int init_vqs(struct virtnet_info *vi); 5711 5712static int virtnet_restore_up(struct virtio_device *vdev) 5713{ 5714 struct virtnet_info *vi = vdev->priv; 5715 int err; 5716 5717 err = init_vqs(vi); 5718 if (err) 5719 return err; 5720 5721 virtio_device_ready(vdev); 5722 5723 enable_rx_mode_work(vi); 5724 5725 if (netif_running(vi->dev)) { 5726 rtnl_lock(); 5727 err = virtnet_open(vi->dev); 5728 rtnl_unlock(); 5729 if (err) 5730 return err; 5731 } 5732 5733 netif_tx_lock_bh(vi->dev); 5734 netif_device_attach(vi->dev); 5735 netif_tx_unlock_bh(vi->dev); 5736 return err; 5737} 5738 5739static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 5740{ 5741 __virtio64 *_offloads __free(kfree) = NULL; 5742 struct scatterlist sg; 5743 5744 _offloads = kzalloc(sizeof(*_offloads), GFP_KERNEL); 5745 if (!_offloads) 5746 return -ENOMEM; 5747 5748 *_offloads = cpu_to_virtio64(vi->vdev, offloads); 5749 5750 sg_init_one(&sg, _offloads, sizeof(*_offloads)); 5751 5752 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 5753 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 5754 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 5755 return -EINVAL; 5756 } 5757 5758 return 0; 5759} 5760 5761static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 5762{ 5763 u64 offloads = 0; 5764 5765 if (!vi->guest_offloads) 5766 return 0; 5767 5768 return virtnet_set_guest_offloads(vi, offloads); 5769} 5770 5771static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 5772{ 5773 u64 offloads = vi->guest_offloads; 5774 5775 if (!vi->guest_offloads) 5776 return 0; 5777 5778 return virtnet_set_guest_offloads(vi, offloads); 5779} 5780 5781static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq, 5782 struct xsk_buff_pool *pool) 5783{ 5784 int err, qindex; 5785 5786 qindex = rq - vi->rq; 5787 5788 if (pool) { 5789 err = xdp_rxq_info_reg(&rq->xsk_rxq_info, vi->dev, qindex, rq->napi.napi_id); 5790 if (err < 0) 5791 return err; 5792 5793 err = xdp_rxq_info_reg_mem_model(&rq->xsk_rxq_info, 5794 MEM_TYPE_XSK_BUFF_POOL, NULL); 5795 if (err < 0) 5796 goto unreg; 5797 5798 xsk_pool_set_rxq_info(pool, &rq->xsk_rxq_info); 5799 } 5800 5801 virtnet_rx_pause(vi, rq); 5802 5803 err = virtqueue_reset(rq->vq, virtnet_rq_unmap_free_buf, NULL); 5804 if (err) { 5805 netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err); 5806 5807 pool = NULL; 5808 } 5809 5810 rq->xsk_pool = pool; 5811 5812 virtnet_rx_resume(vi, rq, true); 5813 5814 if (pool) 5815 return 0; 5816 5817unreg: 5818 xdp_rxq_info_unreg(&rq->xsk_rxq_info); 5819 return err; 5820} 5821 5822static int virtnet_sq_bind_xsk_pool(struct virtnet_info *vi, 5823 struct send_queue *sq, 5824 struct xsk_buff_pool *pool) 5825{ 5826 int err, qindex; 5827 5828 qindex = sq - vi->sq; 5829 5830 virtnet_tx_pause(vi, sq); 5831 5832 err = virtqueue_reset(sq->vq, virtnet_sq_free_unused_buf, 5833 virtnet_sq_free_unused_buf_done); 5834 if (err) { 5835 netdev_err(vi->dev, "reset tx fail: tx queue index: %d err: %d\n", qindex, err); 5836 pool = NULL; 5837 } 5838 5839 sq->xsk_pool = pool; 5840 5841 virtnet_tx_resume(vi, sq); 5842 5843 return err; 5844} 5845 5846static int virtnet_xsk_pool_enable(struct net_device *dev, 5847 struct xsk_buff_pool *pool, 5848 u16 qid) 5849{ 5850 struct virtnet_info *vi = netdev_priv(dev); 5851 struct receive_queue *rq; 5852 struct device *dma_dev; 5853 struct send_queue *sq; 5854 dma_addr_t hdr_dma; 5855 int err, size; 5856 5857 if (vi->hdr_len > xsk_pool_get_headroom(pool)) 5858 return -EINVAL; 5859 5860 /* In big_packets mode, xdp cannot work, so there is no need to 5861 * initialize xsk of rq. 5862 */ 5863 if (vi->big_packets && !vi->mergeable_rx_bufs) 5864 return -ENOENT; 5865 5866 if (qid >= vi->curr_queue_pairs) 5867 return -EINVAL; 5868 5869 sq = &vi->sq[qid]; 5870 rq = &vi->rq[qid]; 5871 5872 /* xsk assumes that tx and rx must have the same dma device. The af-xdp 5873 * may use one buffer to receive from the rx and reuse this buffer to 5874 * send by the tx. So the dma dev of sq and rq must be the same one. 5875 * 5876 * But vq->dma_dev allows every vq has the respective dma dev. So I 5877 * check the dma dev of vq and sq is the same dev. 5878 */ 5879 if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq)) 5880 return -EINVAL; 5881 5882 dma_dev = virtqueue_dma_dev(rq->vq); 5883 if (!dma_dev) 5884 return -EINVAL; 5885 5886 size = virtqueue_get_vring_size(rq->vq); 5887 5888 rq->xsk_buffs = kvcalloc(size, sizeof(*rq->xsk_buffs), GFP_KERNEL); 5889 if (!rq->xsk_buffs) 5890 return -ENOMEM; 5891 5892 hdr_dma = virtqueue_map_single_attrs(sq->vq, &xsk_hdr, vi->hdr_len, 5893 DMA_TO_DEVICE, 0); 5894 if (virtqueue_map_mapping_error(sq->vq, hdr_dma)) { 5895 err = -ENOMEM; 5896 goto err_free_buffs; 5897 } 5898 5899 err = xsk_pool_dma_map(pool, dma_dev, 0); 5900 if (err) 5901 goto err_xsk_map; 5902 5903 err = virtnet_rq_bind_xsk_pool(vi, rq, pool); 5904 if (err) 5905 goto err_rq; 5906 5907 err = virtnet_sq_bind_xsk_pool(vi, sq, pool); 5908 if (err) 5909 goto err_sq; 5910 5911 /* Now, we do not support tx offload(such as tx csum), so all the tx 5912 * virtnet hdr is zero. So all the tx packets can share a single hdr. 5913 */ 5914 sq->xsk_hdr_dma_addr = hdr_dma; 5915 5916 return 0; 5917 5918err_sq: 5919 virtnet_rq_bind_xsk_pool(vi, rq, NULL); 5920err_rq: 5921 xsk_pool_dma_unmap(pool, 0); 5922err_xsk_map: 5923 virtqueue_unmap_single_attrs(rq->vq, hdr_dma, vi->hdr_len, 5924 DMA_TO_DEVICE, 0); 5925err_free_buffs: 5926 kvfree(rq->xsk_buffs); 5927 return err; 5928} 5929 5930static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid) 5931{ 5932 struct virtnet_info *vi = netdev_priv(dev); 5933 struct xsk_buff_pool *pool; 5934 struct receive_queue *rq; 5935 struct send_queue *sq; 5936 int err; 5937 5938 if (qid >= vi->curr_queue_pairs) 5939 return -EINVAL; 5940 5941 sq = &vi->sq[qid]; 5942 rq = &vi->rq[qid]; 5943 5944 pool = rq->xsk_pool; 5945 5946 err = virtnet_rq_bind_xsk_pool(vi, rq, NULL); 5947 err |= virtnet_sq_bind_xsk_pool(vi, sq, NULL); 5948 5949 xsk_pool_dma_unmap(pool, 0); 5950 5951 virtqueue_unmap_single_attrs(sq->vq, sq->xsk_hdr_dma_addr, 5952 vi->hdr_len, DMA_TO_DEVICE, 0); 5953 kvfree(rq->xsk_buffs); 5954 5955 return err; 5956} 5957 5958static int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp) 5959{ 5960 if (xdp->xsk.pool) 5961 return virtnet_xsk_pool_enable(dev, xdp->xsk.pool, 5962 xdp->xsk.queue_id); 5963 else 5964 return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id); 5965} 5966 5967static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 5968 struct netlink_ext_ack *extack) 5969{ 5970 unsigned int room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM + 5971 sizeof(struct skb_shared_info)); 5972 unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN; 5973 struct virtnet_info *vi = netdev_priv(dev); 5974 struct bpf_prog *old_prog; 5975 u16 xdp_qp = 0, curr_qp; 5976 int i, err; 5977 5978 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 5979 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 5980 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 5981 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 5982 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 5983 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || 5984 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || 5985 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { 5986 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 5987 return -EOPNOTSUPP; 5988 } 5989 5990 if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 5991 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 5992 return -EINVAL; 5993 } 5994 5995 if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) { 5996 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags"); 5997 netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz); 5998 return -EINVAL; 5999 } 6000 6001 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 6002 if (prog) 6003 xdp_qp = nr_cpu_ids; 6004 6005 /* XDP requires extra queues for XDP_TX */ 6006 if (curr_qp + xdp_qp > vi->max_queue_pairs) { 6007 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 6008 curr_qp + xdp_qp, vi->max_queue_pairs); 6009 xdp_qp = 0; 6010 } 6011 6012 old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 6013 if (!prog && !old_prog) 6014 return 0; 6015 6016 if (prog) 6017 bpf_prog_add(prog, vi->max_queue_pairs - 1); 6018 6019 virtnet_rx_pause_all(vi); 6020 6021 /* Make sure NAPI is not using any XDP TX queues for RX. */ 6022 if (netif_running(dev)) { 6023 for (i = 0; i < vi->max_queue_pairs; i++) 6024 virtnet_napi_tx_disable(&vi->sq[i]); 6025 } 6026 6027 if (!prog) { 6028 for (i = 0; i < vi->max_queue_pairs; i++) { 6029 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 6030 if (i == 0) 6031 virtnet_restore_guest_offloads(vi); 6032 } 6033 synchronize_net(); 6034 } 6035 6036 err = virtnet_set_queues(vi, curr_qp + xdp_qp); 6037 if (err) 6038 goto err; 6039 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 6040 vi->xdp_queue_pairs = xdp_qp; 6041 6042 if (prog) { 6043 vi->xdp_enabled = true; 6044 for (i = 0; i < vi->max_queue_pairs; i++) { 6045 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 6046 if (i == 0 && !old_prog) 6047 virtnet_clear_guest_offloads(vi); 6048 } 6049 if (!old_prog) 6050 xdp_features_set_redirect_target(dev, true); 6051 } else { 6052 xdp_features_clear_redirect_target(dev); 6053 vi->xdp_enabled = false; 6054 } 6055 6056 virtnet_rx_resume_all(vi); 6057 for (i = 0; i < vi->max_queue_pairs; i++) { 6058 if (old_prog) 6059 bpf_prog_put(old_prog); 6060 if (netif_running(dev)) 6061 virtnet_napi_tx_enable(&vi->sq[i]); 6062 } 6063 6064 return 0; 6065 6066err: 6067 if (!prog) { 6068 virtnet_clear_guest_offloads(vi); 6069 for (i = 0; i < vi->max_queue_pairs; i++) 6070 rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 6071 } 6072 6073 virtnet_rx_resume_all(vi); 6074 if (netif_running(dev)) { 6075 for (i = 0; i < vi->max_queue_pairs; i++) 6076 virtnet_napi_tx_enable(&vi->sq[i]); 6077 } 6078 if (prog) 6079 bpf_prog_sub(prog, vi->max_queue_pairs - 1); 6080 return err; 6081} 6082 6083static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 6084{ 6085 switch (xdp->command) { 6086 case XDP_SETUP_PROG: 6087 return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 6088 case XDP_SETUP_XSK_POOL: 6089 return virtnet_xsk_pool_setup(dev, xdp); 6090 default: 6091 return -EINVAL; 6092 } 6093} 6094 6095static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 6096 size_t len) 6097{ 6098 struct virtnet_info *vi = netdev_priv(dev); 6099 int ret; 6100 6101 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 6102 return -EOPNOTSUPP; 6103 6104 ret = snprintf(buf, len, "sby"); 6105 if (ret >= len) 6106 return -EOPNOTSUPP; 6107 6108 return 0; 6109} 6110 6111static int virtnet_set_features(struct net_device *dev, 6112 netdev_features_t features) 6113{ 6114 struct virtnet_info *vi = netdev_priv(dev); 6115 u64 offloads; 6116 int err; 6117 6118 if ((dev->features ^ features) & NETIF_F_GRO_HW) { 6119 if (vi->xdp_enabled) 6120 return -EBUSY; 6121 6122 if (features & NETIF_F_GRO_HW) 6123 offloads = vi->guest_offloads_capable; 6124 else 6125 offloads = vi->guest_offloads_capable & 6126 ~GUEST_OFFLOAD_GRO_HW_MASK; 6127 6128 err = virtnet_set_guest_offloads(vi, offloads); 6129 if (err) 6130 return err; 6131 vi->guest_offloads = offloads; 6132 } 6133 6134 if ((dev->features ^ features) & NETIF_F_RXHASH) { 6135 if (features & NETIF_F_RXHASH) 6136 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); 6137 else 6138 vi->rss_hdr->hash_types = cpu_to_le32(VIRTIO_NET_HASH_REPORT_NONE); 6139 6140 if (!virtnet_commit_rss_command(vi)) 6141 return -EINVAL; 6142 } 6143 6144 return 0; 6145} 6146 6147static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) 6148{ 6149 struct virtnet_info *priv = netdev_priv(dev); 6150 struct send_queue *sq = &priv->sq[txqueue]; 6151 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); 6152 6153 u64_stats_update_begin(&sq->stats.syncp); 6154 u64_stats_inc(&sq->stats.tx_timeouts); 6155 u64_stats_update_end(&sq->stats.syncp); 6156 6157 netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", 6158 txqueue, sq->name, sq->vq->index, sq->vq->name, 6159 jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); 6160} 6161 6162static int virtnet_init_irq_moder(struct virtnet_info *vi) 6163{ 6164 u8 profile_flags = 0, coal_flags = 0; 6165 int ret, i; 6166 6167 profile_flags |= DIM_PROFILE_RX; 6168 coal_flags |= DIM_COALESCE_USEC | DIM_COALESCE_PKTS; 6169 ret = net_dim_init_irq_moder(vi->dev, profile_flags, coal_flags, 6170 DIM_CQ_PERIOD_MODE_START_FROM_EQE, 6171 0, virtnet_rx_dim_work, NULL); 6172 6173 if (ret) 6174 return ret; 6175 6176 for (i = 0; i < vi->max_queue_pairs; i++) 6177 net_dim_setting(vi->dev, &vi->rq[i].dim, false); 6178 6179 return 0; 6180} 6181 6182static void virtnet_free_irq_moder(struct virtnet_info *vi) 6183{ 6184 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 6185 return; 6186 6187 rtnl_lock(); 6188 net_dim_free_irq_moder(vi->dev); 6189 rtnl_unlock(); 6190} 6191 6192static const struct net_device_ops virtnet_netdev = { 6193 .ndo_open = virtnet_open, 6194 .ndo_stop = virtnet_close, 6195 .ndo_start_xmit = start_xmit, 6196 .ndo_validate_addr = eth_validate_addr, 6197 .ndo_set_mac_address = virtnet_set_mac_address, 6198 .ndo_set_rx_mode = virtnet_set_rx_mode, 6199 .ndo_get_stats64 = virtnet_stats, 6200 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 6201 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 6202 .ndo_bpf = virtnet_xdp, 6203 .ndo_xdp_xmit = virtnet_xdp_xmit, 6204 .ndo_xsk_wakeup = virtnet_xsk_wakeup, 6205 .ndo_features_check = passthru_features_check, 6206 .ndo_get_phys_port_name = virtnet_get_phys_port_name, 6207 .ndo_set_features = virtnet_set_features, 6208 .ndo_tx_timeout = virtnet_tx_timeout, 6209}; 6210 6211static void virtnet_config_changed_work(struct work_struct *work) 6212{ 6213 struct virtnet_info *vi = 6214 container_of(work, struct virtnet_info, config_work); 6215 u16 v; 6216 6217 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 6218 struct virtio_net_config, status, &v) < 0) 6219 return; 6220 6221 if (v & VIRTIO_NET_S_ANNOUNCE) { 6222 netdev_notify_peers(vi->dev); 6223 virtnet_ack_link_announce(vi); 6224 } 6225 6226 /* Ignore unknown (future) status bits */ 6227 v &= VIRTIO_NET_S_LINK_UP; 6228 6229 if (vi->status == v) 6230 return; 6231 6232 vi->status = v; 6233 6234 if (vi->status & VIRTIO_NET_S_LINK_UP) { 6235 virtnet_update_settings(vi); 6236 netif_carrier_on(vi->dev); 6237 netif_tx_wake_all_queues(vi->dev); 6238 } else { 6239 netif_carrier_off(vi->dev); 6240 netif_tx_stop_all_queues(vi->dev); 6241 } 6242} 6243 6244static void virtnet_config_changed(struct virtio_device *vdev) 6245{ 6246 struct virtnet_info *vi = vdev->priv; 6247 6248 schedule_work(&vi->config_work); 6249} 6250 6251static void virtnet_free_queues(struct virtnet_info *vi) 6252{ 6253 int i; 6254 6255 for (i = 0; i < vi->max_queue_pairs; i++) { 6256 __netif_napi_del(&vi->rq[i].napi); 6257 __netif_napi_del(&vi->sq[i].napi); 6258 } 6259 6260 /* We called __netif_napi_del(), 6261 * we need to respect an RCU grace period before freeing vi->rq 6262 */ 6263 synchronize_net(); 6264 6265 kfree(vi->rq); 6266 kfree(vi->sq); 6267 kfree(vi->ctrl); 6268} 6269 6270static void _free_receive_bufs(struct virtnet_info *vi) 6271{ 6272 struct bpf_prog *old_prog; 6273 int i; 6274 6275 for (i = 0; i < vi->max_queue_pairs; i++) { 6276 while (vi->rq[i].pages) 6277 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 6278 6279 old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 6280 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 6281 if (old_prog) 6282 bpf_prog_put(old_prog); 6283 } 6284} 6285 6286static void free_receive_bufs(struct virtnet_info *vi) 6287{ 6288 rtnl_lock(); 6289 _free_receive_bufs(vi); 6290 rtnl_unlock(); 6291} 6292 6293static void free_receive_page_frags(struct virtnet_info *vi) 6294{ 6295 int i; 6296 for (i = 0; i < vi->max_queue_pairs; i++) 6297 if (vi->rq[i].alloc_frag.page) { 6298 if (vi->rq[i].last_dma) 6299 virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0); 6300 put_page(vi->rq[i].alloc_frag.page); 6301 } 6302} 6303 6304static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) 6305{ 6306 struct virtnet_info *vi = vq->vdev->priv; 6307 struct send_queue *sq; 6308 int i = vq2txq(vq); 6309 6310 sq = &vi->sq[i]; 6311 6312 switch (virtnet_xmit_ptr_unpack(&buf)) { 6313 case VIRTNET_XMIT_TYPE_SKB: 6314 case VIRTNET_XMIT_TYPE_SKB_ORPHAN: 6315 dev_kfree_skb(buf); 6316 break; 6317 6318 case VIRTNET_XMIT_TYPE_XDP: 6319 xdp_return_frame(buf); 6320 break; 6321 6322 case VIRTNET_XMIT_TYPE_XSK: 6323 xsk_tx_completed(sq->xsk_pool, 1); 6324 break; 6325 } 6326} 6327 6328static void virtnet_sq_free_unused_buf_done(struct virtqueue *vq) 6329{ 6330 struct virtnet_info *vi = vq->vdev->priv; 6331 int i = vq2txq(vq); 6332 6333 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); 6334} 6335 6336static void free_unused_bufs(struct virtnet_info *vi) 6337{ 6338 void *buf; 6339 int i; 6340 6341 for (i = 0; i < vi->max_queue_pairs; i++) { 6342 struct virtqueue *vq = vi->sq[i].vq; 6343 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 6344 virtnet_sq_free_unused_buf(vq, buf); 6345 cond_resched(); 6346 } 6347 6348 for (i = 0; i < vi->max_queue_pairs; i++) { 6349 struct virtqueue *vq = vi->rq[i].vq; 6350 6351 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 6352 virtnet_rq_unmap_free_buf(vq, buf); 6353 cond_resched(); 6354 } 6355} 6356 6357static void virtnet_del_vqs(struct virtnet_info *vi) 6358{ 6359 struct virtio_device *vdev = vi->vdev; 6360 6361 virtnet_clean_affinity(vi); 6362 6363 vdev->config->del_vqs(vdev); 6364 6365 virtnet_free_queues(vi); 6366} 6367 6368/* How large should a single buffer be so a queue full of these can fit at 6369 * least one full packet? 6370 * Logic below assumes the mergeable buffer header is used. 6371 */ 6372static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 6373{ 6374 const unsigned int hdr_len = vi->hdr_len; 6375 unsigned int rq_size = virtqueue_get_vring_size(vq); 6376 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 6377 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 6378 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 6379 6380 return max(max(min_buf_len, hdr_len) - hdr_len, 6381 (unsigned int)GOOD_PACKET_LEN); 6382} 6383 6384static int virtnet_find_vqs(struct virtnet_info *vi) 6385{ 6386 struct virtqueue_info *vqs_info; 6387 struct virtqueue **vqs; 6388 int ret = -ENOMEM; 6389 int total_vqs; 6390 bool *ctx; 6391 u16 i; 6392 6393 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 6394 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 6395 * possible control vq. 6396 */ 6397 total_vqs = vi->max_queue_pairs * 2 + 6398 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 6399 6400 /* Allocate space for find_vqs parameters */ 6401 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 6402 if (!vqs) 6403 goto err_vq; 6404 vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL); 6405 if (!vqs_info) 6406 goto err_vqs_info; 6407 if (!vi->big_packets || vi->mergeable_rx_bufs) { 6408 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 6409 if (!ctx) 6410 goto err_ctx; 6411 } else { 6412 ctx = NULL; 6413 } 6414 6415 /* Parameters for control virtqueue, if any */ 6416 if (vi->has_cvq) { 6417 vqs_info[total_vqs - 1].name = "control"; 6418 } 6419 6420 /* Allocate/initialize parameters for send/receive virtqueues */ 6421 for (i = 0; i < vi->max_queue_pairs; i++) { 6422 vqs_info[rxq2vq(i)].callback = skb_recv_done; 6423 vqs_info[txq2vq(i)].callback = skb_xmit_done; 6424 sprintf(vi->rq[i].name, "input.%u", i); 6425 sprintf(vi->sq[i].name, "output.%u", i); 6426 vqs_info[rxq2vq(i)].name = vi->rq[i].name; 6427 vqs_info[txq2vq(i)].name = vi->sq[i].name; 6428 if (ctx) 6429 vqs_info[rxq2vq(i)].ctx = true; 6430 } 6431 6432 ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); 6433 if (ret) 6434 goto err_find; 6435 6436 if (vi->has_cvq) { 6437 vi->cvq = vqs[total_vqs - 1]; 6438 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 6439 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 6440 } 6441 6442 for (i = 0; i < vi->max_queue_pairs; i++) { 6443 vi->rq[i].vq = vqs[rxq2vq(i)]; 6444 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 6445 vi->sq[i].vq = vqs[txq2vq(i)]; 6446 } 6447 6448 /* run here: ret == 0. */ 6449 6450 6451err_find: 6452 kfree(ctx); 6453err_ctx: 6454 kfree(vqs_info); 6455err_vqs_info: 6456 kfree(vqs); 6457err_vq: 6458 return ret; 6459} 6460 6461static int virtnet_alloc_queues(struct virtnet_info *vi) 6462{ 6463 int i; 6464 6465 if (vi->has_cvq) { 6466 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 6467 if (!vi->ctrl) 6468 goto err_ctrl; 6469 } else { 6470 vi->ctrl = NULL; 6471 } 6472 vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 6473 if (!vi->sq) 6474 goto err_sq; 6475 vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 6476 if (!vi->rq) 6477 goto err_rq; 6478 6479 for (i = 0; i < vi->max_queue_pairs; i++) { 6480 vi->rq[i].pages = NULL; 6481 netif_napi_add_config(vi->dev, &vi->rq[i].napi, virtnet_poll, 6482 i); 6483 vi->rq[i].napi.weight = napi_weight; 6484 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi, 6485 virtnet_poll_tx, 6486 napi_tx ? napi_weight : 0); 6487 6488 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 6489 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 6490 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 6491 6492 u64_stats_init(&vi->rq[i].stats.syncp); 6493 u64_stats_init(&vi->sq[i].stats.syncp); 6494 mutex_init(&vi->rq[i].dim_lock); 6495 } 6496 6497 return 0; 6498 6499err_rq: 6500 kfree(vi->sq); 6501err_sq: 6502 kfree(vi->ctrl); 6503err_ctrl: 6504 return -ENOMEM; 6505} 6506 6507static int init_vqs(struct virtnet_info *vi) 6508{ 6509 int ret; 6510 6511 /* Allocate send & receive queues */ 6512 ret = virtnet_alloc_queues(vi); 6513 if (ret) 6514 goto err; 6515 6516 ret = virtnet_find_vqs(vi); 6517 if (ret) 6518 goto err_free; 6519 6520 cpus_read_lock(); 6521 virtnet_set_affinity(vi); 6522 cpus_read_unlock(); 6523 6524 return 0; 6525 6526err_free: 6527 virtnet_free_queues(vi); 6528err: 6529 return ret; 6530} 6531 6532#ifdef CONFIG_SYSFS 6533static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 6534 char *buf) 6535{ 6536 struct virtnet_info *vi = netdev_priv(queue->dev); 6537 unsigned int queue_index = get_netdev_rx_queue_index(queue); 6538 unsigned int headroom = virtnet_get_headroom(vi); 6539 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 6540 struct ewma_pkt_len *avg; 6541 6542 BUG_ON(queue_index >= vi->max_queue_pairs); 6543 avg = &vi->rq[queue_index].mrg_avg_pkt_len; 6544 return sprintf(buf, "%u\n", 6545 get_mergeable_buf_len(&vi->rq[queue_index], avg, 6546 SKB_DATA_ALIGN(headroom + tailroom))); 6547} 6548 6549static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 6550 __ATTR_RO(mergeable_rx_buffer_size); 6551 6552static struct attribute *virtio_net_mrg_rx_attrs[] = { 6553 &mergeable_rx_buffer_size_attribute.attr, 6554 NULL 6555}; 6556 6557static const struct attribute_group virtio_net_mrg_rx_group = { 6558 .name = "virtio_net", 6559 .attrs = virtio_net_mrg_rx_attrs 6560}; 6561#endif 6562 6563static bool virtnet_fail_on_feature(struct virtio_device *vdev, 6564 unsigned int fbit, 6565 const char *fname, const char *dname) 6566{ 6567 if (!virtio_has_feature(vdev, fbit)) 6568 return false; 6569 6570 dev_err(&vdev->dev, "device advertises feature %s but not %s", 6571 fname, dname); 6572 6573 return true; 6574} 6575 6576#define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 6577 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 6578 6579static bool virtnet_validate_features(struct virtio_device *vdev) 6580{ 6581 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 6582 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 6583 "VIRTIO_NET_F_CTRL_VQ") || 6584 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 6585 "VIRTIO_NET_F_CTRL_VQ") || 6586 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 6587 "VIRTIO_NET_F_CTRL_VQ") || 6588 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 6589 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 6590 "VIRTIO_NET_F_CTRL_VQ") || 6591 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, 6592 "VIRTIO_NET_F_CTRL_VQ") || 6593 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, 6594 "VIRTIO_NET_F_CTRL_VQ") || 6595 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, 6596 "VIRTIO_NET_F_CTRL_VQ") || 6597 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL, 6598 "VIRTIO_NET_F_CTRL_VQ"))) { 6599 return false; 6600 } 6601 6602 return true; 6603} 6604 6605#define MIN_MTU ETH_MIN_MTU 6606#define MAX_MTU ETH_MAX_MTU 6607 6608static int virtnet_validate(struct virtio_device *vdev) 6609{ 6610 if (!vdev->config->get) { 6611 dev_err(&vdev->dev, "%s failure: config access disabled\n", 6612 __func__); 6613 return -EINVAL; 6614 } 6615 6616 if (!virtnet_validate_features(vdev)) 6617 return -EINVAL; 6618 6619 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 6620 int mtu = virtio_cread16(vdev, 6621 offsetof(struct virtio_net_config, 6622 mtu)); 6623 if (mtu < MIN_MTU) 6624 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 6625 } 6626 6627 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) && 6628 !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 6629 dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby"); 6630 __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY); 6631 } 6632 6633 return 0; 6634} 6635 6636static bool virtnet_check_guest_gso(const struct virtnet_info *vi) 6637{ 6638 return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 6639 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 6640 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 6641 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 6642 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && 6643 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); 6644} 6645 6646static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) 6647{ 6648 bool guest_gso = virtnet_check_guest_gso(vi); 6649 6650 /* If device can receive ANY guest GSO packets, regardless of mtu, 6651 * allocate packets of maximum size, otherwise limit it to only 6652 * mtu size worth only. 6653 */ 6654 if (mtu > ETH_DATA_LEN || guest_gso) { 6655 vi->big_packets = true; 6656 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); 6657 } 6658} 6659 6660#define VIRTIO_NET_HASH_REPORT_MAX_TABLE 10 6661static enum xdp_rss_hash_type 6662virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = { 6663 [VIRTIO_NET_HASH_REPORT_NONE] = XDP_RSS_TYPE_NONE, 6664 [VIRTIO_NET_HASH_REPORT_IPv4] = XDP_RSS_TYPE_L3_IPV4, 6665 [VIRTIO_NET_HASH_REPORT_TCPv4] = XDP_RSS_TYPE_L4_IPV4_TCP, 6666 [VIRTIO_NET_HASH_REPORT_UDPv4] = XDP_RSS_TYPE_L4_IPV4_UDP, 6667 [VIRTIO_NET_HASH_REPORT_IPv6] = XDP_RSS_TYPE_L3_IPV6, 6668 [VIRTIO_NET_HASH_REPORT_TCPv6] = XDP_RSS_TYPE_L4_IPV6_TCP, 6669 [VIRTIO_NET_HASH_REPORT_UDPv6] = XDP_RSS_TYPE_L4_IPV6_UDP, 6670 [VIRTIO_NET_HASH_REPORT_IPv6_EX] = XDP_RSS_TYPE_L3_IPV6_EX, 6671 [VIRTIO_NET_HASH_REPORT_TCPv6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX, 6672 [VIRTIO_NET_HASH_REPORT_UDPv6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX 6673}; 6674 6675static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, 6676 enum xdp_rss_hash_type *rss_type) 6677{ 6678 const struct xdp_buff *xdp = (void *)_ctx; 6679 struct virtio_net_hdr_v1_hash *hdr_hash; 6680 struct virtnet_info *vi; 6681 u16 hash_report; 6682 6683 if (!(xdp->rxq->dev->features & NETIF_F_RXHASH)) 6684 return -ENODATA; 6685 6686 vi = netdev_priv(xdp->rxq->dev); 6687 hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len); 6688 hash_report = __le16_to_cpu(hdr_hash->hash_report); 6689 6690 if (hash_report >= VIRTIO_NET_HASH_REPORT_MAX_TABLE) 6691 hash_report = VIRTIO_NET_HASH_REPORT_NONE; 6692 6693 *rss_type = virtnet_xdp_rss_type[hash_report]; 6694 *hash = virtio_net_hash_value(hdr_hash); 6695 return 0; 6696} 6697 6698static const struct xdp_metadata_ops virtnet_xdp_metadata_ops = { 6699 .xmo_rx_hash = virtnet_xdp_rx_hash, 6700}; 6701 6702static int virtnet_probe(struct virtio_device *vdev) 6703{ 6704 int i, err = -ENOMEM; 6705 struct net_device *dev; 6706 struct virtnet_info *vi; 6707 u16 max_queue_pairs; 6708 int mtu = 0; 6709 u16 key_sz; 6710 6711 /* Find if host supports multiqueue/rss virtio_net device */ 6712 max_queue_pairs = 1; 6713 if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 6714 max_queue_pairs = 6715 virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); 6716 6717 /* We need at least 2 queue's */ 6718 if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 6719 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 6720 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 6721 max_queue_pairs = 1; 6722 6723 /* Allocate ourselves a network device with room for our info */ 6724 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 6725 if (!dev) 6726 return -ENOMEM; 6727 6728 /* Set up network device as normal. */ 6729 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 6730 IFF_TX_SKB_NO_LINEAR; 6731 dev->netdev_ops = &virtnet_netdev; 6732 dev->stat_ops = &virtnet_stat_ops; 6733 dev->features = NETIF_F_HIGHDMA; 6734 6735 dev->ethtool_ops = &virtnet_ethtool_ops; 6736 SET_NETDEV_DEV(dev, &vdev->dev); 6737 6738 /* Do we support "hardware" checksums? */ 6739 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 6740 /* This opens up the world of extra features. */ 6741 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 6742 if (csum) 6743 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 6744 6745 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 6746 dev->hw_features |= NETIF_F_TSO 6747 | NETIF_F_TSO_ECN | NETIF_F_TSO6; 6748 } 6749 /* Individual feature bits: what can host handle? */ 6750 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 6751 dev->hw_features |= NETIF_F_TSO; 6752 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 6753 dev->hw_features |= NETIF_F_TSO6; 6754 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 6755 dev->hw_features |= NETIF_F_TSO_ECN; 6756 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO)) 6757 dev->hw_features |= NETIF_F_GSO_UDP_L4; 6758 6759 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO)) { 6760 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 6761 dev->hw_enc_features = dev->hw_features; 6762 } 6763 if (dev->hw_features & NETIF_F_GSO_UDP_TUNNEL && 6764 virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO_CSUM)) { 6765 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 6766 dev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 6767 } 6768 6769 dev->features |= NETIF_F_GSO_ROBUST; 6770 6771 if (gso) 6772 dev->features |= dev->hw_features; 6773 /* (!csum && gso) case will be fixed by register_netdev() */ 6774 } 6775 6776 /* 1. With VIRTIO_NET_F_GUEST_CSUM negotiation, the driver doesn't 6777 * need to calculate checksums for partially checksummed packets, 6778 * as they're considered valid by the upper layer. 6779 * 2. Without VIRTIO_NET_F_GUEST_CSUM negotiation, the driver only 6780 * receives fully checksummed packets. The device may assist in 6781 * validating these packets' checksums, so the driver won't have to. 6782 */ 6783 dev->features |= NETIF_F_RXCSUM; 6784 6785 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 6786 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 6787 dev->features |= NETIF_F_GRO_HW; 6788 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 6789 dev->hw_features |= NETIF_F_GRO_HW; 6790 6791 dev->vlan_features = dev->features; 6792 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 6793 NETDEV_XDP_ACT_XSK_ZEROCOPY; 6794 6795 /* MTU range: 68 - 65535 */ 6796 dev->min_mtu = MIN_MTU; 6797 dev->max_mtu = MAX_MTU; 6798 6799 /* Configuration may specify what MAC to use. Otherwise random. */ 6800 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 6801 u8 addr[ETH_ALEN]; 6802 6803 virtio_cread_bytes(vdev, 6804 offsetof(struct virtio_net_config, mac), 6805 addr, ETH_ALEN); 6806 eth_hw_addr_set(dev, addr); 6807 } else { 6808 eth_hw_addr_random(dev); 6809 dev_info(&vdev->dev, "Assigned random MAC address %pM\n", 6810 dev->dev_addr); 6811 } 6812 6813 /* Set up our device-specific information */ 6814 vi = netdev_priv(dev); 6815 vi->dev = dev; 6816 vi->vdev = vdev; 6817 vdev->priv = vi; 6818 6819 INIT_WORK(&vi->config_work, virtnet_config_changed_work); 6820 INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work); 6821 6822 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { 6823 vi->mergeable_rx_bufs = true; 6824 dev->xdp_features |= NETDEV_XDP_ACT_RX_SG; 6825 } 6826 6827 if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) 6828 vi->has_rss_hash_report = true; 6829 6830 if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) { 6831 vi->has_rss = true; 6832 6833 vi->rss_indir_table_size = 6834 virtio_cread16(vdev, offsetof(struct virtio_net_config, 6835 rss_max_indirection_table_length)); 6836 } 6837 vi->rss_hdr = devm_kzalloc(&vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); 6838 if (!vi->rss_hdr) { 6839 err = -ENOMEM; 6840 goto free; 6841 } 6842 6843 if (vi->has_rss || vi->has_rss_hash_report) { 6844 key_sz = virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); 6845 6846 vi->rss_key_size = min_t(u16, key_sz, NETDEV_RSS_KEY_LEN); 6847 if (key_sz > vi->rss_key_size) 6848 dev_warn(&vdev->dev, 6849 "rss_max_key_size=%u exceeds driver limit %u, clamping\n", 6850 key_sz, vi->rss_key_size); 6851 6852 vi->rss_hash_types_supported = 6853 virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); 6854 vi->rss_hash_types_supported &= 6855 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | 6856 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 6857 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); 6858 6859 dev->hw_features |= NETIF_F_RXHASH; 6860 dev->xdp_metadata_ops = &virtnet_xdp_metadata_ops; 6861 } 6862 6863 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO) || 6864 virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO)) 6865 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash_tunnel); 6866 else if (vi->has_rss_hash_report) 6867 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); 6868 else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 6869 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 6870 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 6871 else 6872 vi->hdr_len = sizeof(struct virtio_net_hdr); 6873 6874 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM)) 6875 vi->rx_tnl_csum = true; 6876 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO)) 6877 vi->rx_tnl = true; 6878 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO)) 6879 vi->tx_tnl = true; 6880 6881 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 6882 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 6883 vi->any_header_sg = true; 6884 6885 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 6886 vi->has_cvq = true; 6887 6888 mutex_init(&vi->cvq_lock); 6889 6890 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 6891 mtu = virtio_cread16(vdev, 6892 offsetof(struct virtio_net_config, 6893 mtu)); 6894 if (mtu < dev->min_mtu) { 6895 /* Should never trigger: MTU was previously validated 6896 * in virtnet_validate. 6897 */ 6898 dev_err(&vdev->dev, 6899 "device MTU appears to have changed it is now %d < %d", 6900 mtu, dev->min_mtu); 6901 err = -EINVAL; 6902 goto free; 6903 } 6904 6905 dev->mtu = mtu; 6906 dev->max_mtu = mtu; 6907 } 6908 6909 virtnet_set_big_packets(vi, mtu); 6910 6911 if (vi->any_header_sg) 6912 dev->needed_headroom = vi->hdr_len; 6913 6914 /* Enable multiqueue by default */ 6915 if (num_online_cpus() >= max_queue_pairs) 6916 vi->curr_queue_pairs = max_queue_pairs; 6917 else 6918 vi->curr_queue_pairs = num_online_cpus(); 6919 vi->max_queue_pairs = max_queue_pairs; 6920 6921 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 6922 err = init_vqs(vi); 6923 if (err) 6924 goto free; 6925 6926 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 6927 vi->intr_coal_rx.max_usecs = 0; 6928 vi->intr_coal_tx.max_usecs = 0; 6929 vi->intr_coal_rx.max_packets = 0; 6930 6931 /* Keep the default values of the coalescing parameters 6932 * aligned with the default napi_tx state. 6933 */ 6934 if (vi->sq[0].napi.weight) 6935 vi->intr_coal_tx.max_packets = 1; 6936 else 6937 vi->intr_coal_tx.max_packets = 0; 6938 } 6939 6940 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { 6941 /* The reason is the same as VIRTIO_NET_F_NOTF_COAL. */ 6942 for (i = 0; i < vi->max_queue_pairs; i++) 6943 if (vi->sq[i].napi.weight) 6944 vi->sq[i].intr_coal.max_packets = 1; 6945 6946 err = virtnet_init_irq_moder(vi); 6947 if (err) 6948 goto free; 6949 } 6950 6951#ifdef CONFIG_SYSFS 6952 if (vi->mergeable_rx_bufs) 6953 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 6954#endif 6955 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 6956 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 6957 6958 virtnet_init_settings(dev); 6959 6960 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 6961 vi->failover = net_failover_create(vi->dev); 6962 if (IS_ERR(vi->failover)) { 6963 err = PTR_ERR(vi->failover); 6964 goto free_vqs; 6965 } 6966 } 6967 6968 if (vi->has_rss || vi->has_rss_hash_report) 6969 virtnet_init_default_rss(vi); 6970 6971 enable_rx_mode_work(vi); 6972 6973 /* serialize netdev register + virtio_device_ready() with ndo_open() */ 6974 rtnl_lock(); 6975 6976 err = register_netdevice(dev); 6977 if (err) { 6978 pr_debug("virtio_net: registering device failed\n"); 6979 rtnl_unlock(); 6980 goto free_failover; 6981 } 6982 6983 /* Disable config change notification until ndo_open. */ 6984 virtio_config_driver_disable(vi->vdev); 6985 6986 virtio_device_ready(vdev); 6987 6988 if (vi->has_rss || vi->has_rss_hash_report) { 6989 if (!virtnet_commit_rss_command(vi)) { 6990 dev_warn(&vdev->dev, "RSS disabled because committing failed.\n"); 6991 dev->hw_features &= ~NETIF_F_RXHASH; 6992 vi->has_rss_hash_report = false; 6993 vi->has_rss = false; 6994 } 6995 } 6996 6997 virtnet_set_queues(vi, vi->curr_queue_pairs); 6998 6999 /* a random MAC address has been assigned, notify the device. 7000 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there 7001 * because many devices work fine without getting MAC explicitly 7002 */ 7003 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 7004 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 7005 struct scatterlist sg; 7006 7007 sg_init_one(&sg, dev->dev_addr, dev->addr_len); 7008 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 7009 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 7010 pr_debug("virtio_net: setting MAC address failed\n"); 7011 rtnl_unlock(); 7012 err = -EINVAL; 7013 goto free_unregister_netdev; 7014 } 7015 } 7016 7017 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) { 7018 struct virtio_net_stats_capabilities *stats_cap __free(kfree) = NULL; 7019 struct scatterlist sg; 7020 __le64 v; 7021 7022 stats_cap = kzalloc(sizeof(*stats_cap), GFP_KERNEL); 7023 if (!stats_cap) { 7024 rtnl_unlock(); 7025 err = -ENOMEM; 7026 goto free_unregister_netdev; 7027 } 7028 7029 sg_init_one(&sg, stats_cap, sizeof(*stats_cap)); 7030 7031 if (!virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, 7032 VIRTIO_NET_CTRL_STATS_QUERY, 7033 NULL, &sg)) { 7034 pr_debug("virtio_net: fail to get stats capability\n"); 7035 rtnl_unlock(); 7036 err = -EINVAL; 7037 goto free_unregister_netdev; 7038 } 7039 7040 v = stats_cap->supported_stats_types[0]; 7041 vi->device_stats_cap = le64_to_cpu(v); 7042 } 7043 7044 /* Assume link up if device can't report link status, 7045 otherwise get link status from config. */ 7046 netif_carrier_off(dev); 7047 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 7048 virtio_config_changed(vi->vdev); 7049 } else { 7050 vi->status = VIRTIO_NET_S_LINK_UP; 7051 virtnet_update_settings(vi); 7052 netif_carrier_on(dev); 7053 } 7054 7055 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) { 7056 unsigned int fbit; 7057 7058 fbit = virtio_offload_to_feature(guest_offloads[i]); 7059 if (virtio_has_feature(vi->vdev, fbit)) 7060 set_bit(guest_offloads[i], &vi->guest_offloads); 7061 } 7062 vi->guest_offloads_capable = vi->guest_offloads; 7063 7064 rtnl_unlock(); 7065 7066 err = virtnet_cpu_notif_add(vi); 7067 if (err) { 7068 pr_debug("virtio_net: registering cpu notifier failed\n"); 7069 goto free_unregister_netdev; 7070 } 7071 7072 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 7073 dev->name, max_queue_pairs); 7074 7075 return 0; 7076 7077free_unregister_netdev: 7078 unregister_netdev(dev); 7079free_failover: 7080 net_failover_destroy(vi->failover); 7081free_vqs: 7082 virtio_reset_device(vdev); 7083 free_receive_page_frags(vi); 7084 virtnet_del_vqs(vi); 7085free: 7086 free_netdev(dev); 7087 return err; 7088} 7089 7090static void remove_vq_common(struct virtnet_info *vi) 7091{ 7092 int i; 7093 7094 virtio_reset_device(vi->vdev); 7095 7096 /* Free unused buffers in both send and recv, if any. */ 7097 free_unused_bufs(vi); 7098 7099 /* 7100 * Rule of thumb is netdev_tx_reset_queue() should follow any 7101 * skb freeing not followed by netdev_tx_completed_queue() 7102 */ 7103 for (i = 0; i < vi->max_queue_pairs; i++) 7104 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); 7105 7106 free_receive_bufs(vi); 7107 7108 free_receive_page_frags(vi); 7109 7110 virtnet_del_vqs(vi); 7111} 7112 7113static void virtnet_remove(struct virtio_device *vdev) 7114{ 7115 struct virtnet_info *vi = vdev->priv; 7116 7117 virtnet_cpu_notif_remove(vi); 7118 7119 /* Make sure no work handler is accessing the device. */ 7120 flush_work(&vi->config_work); 7121 disable_rx_mode_work(vi); 7122 flush_work(&vi->rx_mode_work); 7123 7124 virtnet_free_irq_moder(vi); 7125 7126 unregister_netdev(vi->dev); 7127 7128 net_failover_destroy(vi->failover); 7129 7130 remove_vq_common(vi); 7131 7132 free_netdev(vi->dev); 7133} 7134 7135static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 7136{ 7137 struct virtnet_info *vi = vdev->priv; 7138 7139 virtnet_cpu_notif_remove(vi); 7140 virtnet_freeze_down(vdev); 7141 remove_vq_common(vi); 7142 7143 return 0; 7144} 7145 7146static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 7147{ 7148 struct virtnet_info *vi = vdev->priv; 7149 int err; 7150 7151 err = virtnet_restore_up(vdev); 7152 if (err) 7153 return err; 7154 virtnet_set_queues(vi, vi->curr_queue_pairs); 7155 7156 err = virtnet_cpu_notif_add(vi); 7157 if (err) { 7158 virtnet_freeze_down(vdev); 7159 remove_vq_common(vi); 7160 return err; 7161 } 7162 7163 return 0; 7164} 7165 7166static struct virtio_device_id id_table[] = { 7167 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 7168 { 0 }, 7169}; 7170 7171#define VIRTNET_FEATURES \ 7172 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 7173 VIRTIO_NET_F_MAC, \ 7174 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 7175 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 7176 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 7177 VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \ 7178 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 7179 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 7180 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 7181 VIRTIO_NET_F_CTRL_MAC_ADDR, \ 7182 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 7183 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ 7184 VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \ 7185 VIRTIO_NET_F_VQ_NOTF_COAL, \ 7186 VIRTIO_NET_F_GUEST_HDRLEN, VIRTIO_NET_F_DEVICE_STATS 7187 7188static unsigned int features[] = { 7189 VIRTNET_FEATURES, 7190 VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO, 7191 VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO_CSUM, 7192 VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO, 7193 VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO_CSUM, 7194}; 7195 7196static unsigned int features_legacy[] = { 7197 VIRTNET_FEATURES, 7198 VIRTIO_NET_F_GSO, 7199 VIRTIO_F_ANY_LAYOUT, 7200}; 7201 7202static struct virtio_driver virtio_net_driver = { 7203 .feature_table = features, 7204 .feature_table_size = ARRAY_SIZE(features), 7205 .feature_table_legacy = features_legacy, 7206 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 7207 .driver.name = KBUILD_MODNAME, 7208 .id_table = id_table, 7209 .validate = virtnet_validate, 7210 .probe = virtnet_probe, 7211 .remove = virtnet_remove, 7212 .config_changed = virtnet_config_changed, 7213#ifdef CONFIG_PM_SLEEP 7214 .freeze = virtnet_freeze, 7215 .restore = virtnet_restore, 7216#endif 7217}; 7218 7219static __init int virtio_net_driver_init(void) 7220{ 7221 int ret; 7222 7223 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 7224 virtnet_cpu_online, 7225 virtnet_cpu_down_prep); 7226 if (ret < 0) 7227 goto out; 7228 virtionet_online = ret; 7229 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 7230 NULL, virtnet_cpu_dead); 7231 if (ret) 7232 goto err_dead; 7233 ret = register_virtio_driver(&virtio_net_driver); 7234 if (ret) 7235 goto err_virtio; 7236 return 0; 7237err_virtio: 7238 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 7239err_dead: 7240 cpuhp_remove_multi_state(virtionet_online); 7241out: 7242 return ret; 7243} 7244module_init(virtio_net_driver_init); 7245 7246static __exit void virtio_net_driver_exit(void) 7247{ 7248 unregister_virtio_driver(&virtio_net_driver); 7249 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 7250 cpuhp_remove_multi_state(virtionet_online); 7251} 7252module_exit(virtio_net_driver_exit); 7253 7254MODULE_DEVICE_TABLE(virtio, id_table); 7255MODULE_DESCRIPTION("Virtio network driver"); 7256MODULE_LICENSE("GPL");
~/
Read-only file preview.