FILE
en_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
1/* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/dim.h> 34#include <net/tc_act/tc_gact.h> 35#include <linux/mlx5/fs.h> 36#include <net/vxlan.h> 37#include <net/geneve.h> 38#include <linux/bpf.h> 39#include <linux/debugfs.h> 40#include <linux/if_bridge.h> 41#include <linux/filter.h> 42#include <net/netdev_lock.h> 43#include <net/netdev_queues.h> 44#include <net/netdev_rx_queue.h> 45#include <net/page_pool/types.h> 46#include <net/pkt_sched.h> 47#include <net/xdp_sock_drv.h> 48#include "eswitch.h" 49#include "en.h" 50#include "en/dim.h" 51#include "en/txrx.h" 52#include "en_tc.h" 53#include "en_rep.h" 54#include "en_accel/ipsec.h" 55#include "en_accel/psp.h" 56#include "en_accel/macsec.h" 57#include "en_accel/en_accel.h" 58#include "en_accel/ktls.h" 59#include "lib/vxlan.h" 60#include "lib/clock.h" 61#include "en/port.h" 62#include "en/xdp.h" 63#include "lib/eq.h" 64#include "en/monitor_stats.h" 65#include "en/health.h" 66#include "en/params.h" 67#include "en/xsk/pool.h" 68#include "en/xsk/setup.h" 69#include "en/xsk/rx.h" 70#include "en/xsk/tx.h" 71#include "en/hv_vhca_stats.h" 72#include "en/devlink.h" 73#include "lib/mlx5.h" 74#include "en/ptp.h" 75#include "en/htb.h" 76#include "qos.h" 77#include "en/trap.h" 78#include "lib/devcom.h" 79#include "lib/sd.h" 80#include "en/pcie_cong_event.h" 81 82static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev) 83{ 84 if (!MLX5_CAP_GEN(mdev, shampo) || 85 !MLX5_CAP_SHAMPO(mdev, shampo_header_split_data_merge)) 86 return false; 87 88 /* Our HW-GRO implementation relies on "KSM Mkey" for 89 * SHAMPO headers buffer mapping 90 */ 91 if (!MLX5_CAP_GEN(mdev, fixed_buffer_size)) 92 return false; 93 94 if (!MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer_valid)) 95 return false; 96 97 if (MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer) > 98 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 99 return false; 100 101 return true; 102} 103 104bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 105 enum mlx5e_mpwrq_umr_mode umr_mode) 106{ 107 u16 umr_wqebbs, max_wqebbs; 108 bool striding_rq_umr; 109 110 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 111 MLX5_CAP_ETH(mdev, reg_umr_sq); 112 if (!striding_rq_umr) 113 return false; 114 115 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); 116 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 117 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is 118 * calculated from mlx5e_get_max_sq_aligned_wqebbs. 119 */ 120 if (WARN_ON(umr_wqebbs > max_wqebbs)) 121 return false; 122 123 return true; 124} 125 126void mlx5e_update_carrier(struct mlx5e_priv *priv) 127{ 128 struct mlx5_core_dev *mdev = priv->mdev; 129 u8 port_state; 130 bool up; 131 132 port_state = mlx5_query_vport_state(mdev, 133 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 134 0); 135 136 up = port_state == VPORT_STATE_UP; 137 if (up == netif_carrier_ok(priv->netdev)) 138 netif_carrier_event(priv->netdev); 139 if (up) { 140 netdev_info(priv->netdev, "Link up\n"); 141 netif_carrier_on(priv->netdev); 142 } else { 143 netdev_info(priv->netdev, "Link down\n"); 144 netif_carrier_off(priv->netdev); 145 } 146} 147 148static void mlx5e_update_carrier_work(struct work_struct *work) 149{ 150 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 151 update_carrier_work); 152 153 mutex_lock(&priv->state_lock); 154 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 155 if (priv->profile->update_carrier) 156 priv->profile->update_carrier(priv); 157 mutex_unlock(&priv->state_lock); 158} 159 160static void mlx5e_update_stats_work(struct work_struct *work) 161{ 162 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 163 update_stats_work); 164 165 mutex_lock(&priv->state_lock); 166 priv->profile->update_stats(priv); 167 mutex_unlock(&priv->state_lock); 168} 169 170void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 171{ 172 if (!priv->profile->update_stats) 173 return; 174 175 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 176 return; 177 178 queue_work(priv->wq, &priv->update_stats_work); 179} 180 181static int async_event(struct notifier_block *nb, unsigned long event, void *data) 182{ 183 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 184 struct mlx5_eqe *eqe = data; 185 186 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 187 return NOTIFY_DONE; 188 189 switch (eqe->sub_type) { 190 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 191 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 192 queue_work(priv->wq, &priv->update_carrier_work); 193 break; 194 default: 195 return NOTIFY_DONE; 196 } 197 198 return NOTIFY_OK; 199} 200 201static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 202{ 203 priv->events_nb.notifier_call = async_event; 204 mlx5_notifier_register(priv->mdev, &priv->events_nb); 205} 206 207static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 208{ 209 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 210} 211 212static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) 213{ 214 struct mlx5e_priv *slave_priv = my_data; 215 216 switch (event) { 217 case MPV_DEVCOM_MASTER_UP: 218 mlx5_devcom_comp_set_ready(slave_priv->devcom, true); 219 break; 220 case MPV_DEVCOM_MASTER_DOWN: 221 /* no need for comp set ready false since we unregister after 222 * and it hurts cleanup flow. 223 */ 224 break; 225 case MPV_DEVCOM_IPSEC_MASTER_UP: 226 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 227 mlx5e_ipsec_handle_mpv_event(event, my_data, event_data); 228 break; 229 } 230 231 return 0; 232} 233 234static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) 235{ 236 struct mlx5_devcom_match_attr attr = { 237 .key.val = *data, 238 }; 239 240 priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc, 241 MLX5_DEVCOM_MPV, 242 &attr, 243 mlx5e_devcom_event_mpv, 244 priv); 245 if (!priv->devcom) 246 return -EINVAL; 247 248 if (mlx5_core_is_mp_master(priv->mdev)) { 249 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP, 250 MPV_DEVCOM_MASTER_UP, priv); 251 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_UP); 252 } 253 254 return 0; 255} 256 257static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv) 258{ 259 if (!priv->devcom) 260 return; 261 262 if (mlx5_core_is_mp_master(priv->mdev)) { 263 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_DOWN, 264 MPV_DEVCOM_MASTER_DOWN, priv); 265 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_DOWN); 266 } 267 268 mlx5_devcom_unregister_component(priv->devcom); 269 priv->devcom = NULL; 270} 271 272static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) 273{ 274 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); 275 struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data; 276 int err; 277 278 switch (event) { 279 case MLX5_DRIVER_EVENT_TYPE_TRAP: 280 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap); 281 if (err) { 282 trap_event_ctx->err = err; 283 return NOTIFY_BAD; 284 } 285 break; 286 case MLX5_DRIVER_EVENT_AFFILIATION_DONE: 287 if (mlx5e_devcom_init_mpv(priv, data)) 288 return NOTIFY_BAD; 289 break; 290 case MLX5_DRIVER_EVENT_AFFILIATION_REMOVED: 291 mlx5e_devcom_cleanup_mpv(priv); 292 break; 293 default: 294 return NOTIFY_DONE; 295 } 296 return NOTIFY_OK; 297} 298 299static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) 300{ 301 priv->blocking_events_nb.notifier_call = blocking_event; 302 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb); 303} 304 305static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) 306{ 307 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); 308} 309 310static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) 311{ 312 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); 313 u32 sz; 314 315 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); 316 317 return sz / MLX5_OCTWORD; 318} 319 320static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 321 struct mlx5e_icosq *sq, 322 struct mlx5e_umr_wqe *wqe) 323{ 324 struct mlx5_wqe_ctrl_seg *cseg = &wqe->hdr.ctrl; 325 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->hdr.uctrl; 326 u16 octowords; 327 u8 ds_cnt; 328 329 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, 330 rq->mpwqe.umr_mode), 331 MLX5_SEND_WQE_DS); 332 333 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 334 ds_cnt); 335 cseg->umr_mkey = rq->mpwqe.umr_mkey_be; 336 337 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 338 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); 339 ucseg->xlt_octowords = cpu_to_be16(octowords); 340 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 341} 342 343static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) 344{ 345 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 346 size_t alloc_size; 347 348 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, 349 alloc_units.frag_pages, 350 rq->mpwqe.pages_per_wqe)); 351 352 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); 353 if (!rq->mpwqe.info) 354 return -ENOMEM; 355 356 /* For deferred page release (release right before alloc), make sure 357 * that on first round release is not called. 358 */ 359 for (int i = 0; i < wq_sz; i++) { 360 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i); 361 362 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); 363 } 364 365 mlx5e_build_umr_wqe(rq, rq->icosq, 366 container_of(&rq->mpwqe.umr_wqe, 367 struct mlx5e_umr_wqe, hdr)); 368 369 return 0; 370} 371 372 373static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) 374{ 375 switch (umr_mode) { 376 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 377 return MLX5_MKC_ACCESS_MODE_MTT; 378 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 379 return MLX5_MKC_ACCESS_MODE_KSM; 380 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 381 return MLX5_MKC_ACCESS_MODE_KLMS; 382 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 383 return MLX5_MKC_ACCESS_MODE_KSM; 384 } 385 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); 386 return 0; 387} 388 389static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 390 u32 npages, u8 page_shift, u32 *umr_mkey, 391 dma_addr_t filler_addr, 392 enum mlx5e_mpwrq_umr_mode umr_mode, 393 u32 xsk_chunk_size) 394{ 395 struct mlx5_mtt *mtt; 396 struct mlx5_ksm *ksm; 397 struct mlx5_klm *klm; 398 u32 octwords; 399 int inlen; 400 void *mkc; 401 u32 *in; 402 int err; 403 int i; 404 405 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || 406 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && 407 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { 408 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); 409 return -EINVAL; 410 } 411 412 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); 413 414 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 415 MLX5_OCTWORD, octwords); 416 if (inlen < 0) 417 return inlen; 418 419 in = kvzalloc(inlen, GFP_KERNEL); 420 if (!in) 421 return -ENOMEM; 422 423 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 424 425 MLX5_SET(mkc, mkc, free, 1); 426 MLX5_SET(mkc, mkc, umr_en, 1); 427 MLX5_SET(mkc, mkc, lw, 1); 428 MLX5_SET(mkc, mkc, lr, 1); 429 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); 430 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 431 MLX5_SET(mkc, mkc, qpn, 0xffffff); 432 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 433 MLX5_SET64(mkc, mkc, len, npages << page_shift); 434 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 435 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) 436 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); 437 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) 438 MLX5_SET(mkc, mkc, log_page_size, page_shift); 439 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); 440 441 /* Initialize the mkey with all MTTs pointing to a default 442 * page (filler_addr). When the channels are activated, UMR 443 * WQEs will redirect the RX WQEs to the actual memory from 444 * the RQ's pool, while the gaps (wqe_overflow) remain mapped 445 * to the default page. 446 */ 447 switch (umr_mode) { 448 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 449 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 450 for (i = 0; i < npages; i++) { 451 klm[i << 1] = (struct mlx5_klm) { 452 .va = cpu_to_be64(filler_addr), 453 .bcount = cpu_to_be32(xsk_chunk_size), 454 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 455 }; 456 klm[(i << 1) + 1] = (struct mlx5_klm) { 457 .va = cpu_to_be64(filler_addr), 458 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), 459 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 460 }; 461 } 462 break; 463 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 464 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 465 for (i = 0; i < npages; i++) 466 ksm[i] = (struct mlx5_ksm) { 467 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 468 .va = cpu_to_be64(filler_addr), 469 }; 470 break; 471 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 472 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 473 for (i = 0; i < npages; i++) 474 mtt[i] = (struct mlx5_mtt) { 475 .ptag = cpu_to_be64(filler_addr), 476 }; 477 break; 478 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 479 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 480 for (i = 0; i < npages * 4; i++) { 481 ksm[i] = (struct mlx5_ksm) { 482 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 483 .va = cpu_to_be64(filler_addr), 484 }; 485 } 486 break; 487 } 488 489 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 490 491 kvfree(in); 492 return err; 493} 494 495static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev, 496 u64 nentries, u8 log_entry_size, 497 u32 *umr_mkey) 498{ 499 int inlen; 500 void *mkc; 501 u32 *in; 502 int err; 503 504 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 505 506 in = kvzalloc(inlen, GFP_KERNEL); 507 if (!in) 508 return -ENOMEM; 509 510 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 511 512 MLX5_SET(mkc, mkc, free, 1); 513 MLX5_SET(mkc, mkc, umr_en, 1); 514 MLX5_SET(mkc, mkc, lw, 1); 515 MLX5_SET(mkc, mkc, lr, 1); 516 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KSM); 517 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 518 MLX5_SET(mkc, mkc, qpn, 0xffffff); 519 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 520 MLX5_SET(mkc, mkc, translations_octword_size, nentries); 521 MLX5_SET(mkc, mkc, log_page_size, log_entry_size); 522 MLX5_SET64(mkc, mkc, len, nentries << log_entry_size); 523 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 524 525 kvfree(in); 526 return err; 527} 528 529static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 530{ 531 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; 532 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 533 u32 num_entries, max_num_entries; 534 u32 umr_mkey; 535 int err; 536 537 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); 538 539 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ 540 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, 541 &num_entries) || 542 num_entries > max_num_entries)) 543 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", 544 __func__, wq_size, rq->mpwqe.mtts_per_wqe, 545 max_num_entries); 546 547 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, 548 &umr_mkey, rq->wqe_overflow.addr, 549 rq->mpwqe.umr_mode, xsk_chunk_size); 550 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 551 return err; 552} 553 554static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, 555 u16 hd_per_wq, __be32 *umr_mkey) 556{ 557 u32 max_ksm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size)); 558 u32 mkey; 559 int err; 560 561 if (max_ksm_size < hd_per_wq) { 562 mlx5_core_err(mdev, "max ksm list size 0x%x is smaller than shampo header buffer list size 0x%x\n", 563 max_ksm_size, hd_per_wq); 564 return -EINVAL; 565 } 566 567 err = mlx5e_create_umr_ksm_mkey(mdev, hd_per_wq, 568 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE, 569 &mkey); 570 if (err) 571 return err; 572 573 *umr_mkey = cpu_to_be32(mkey); 574 return 0; 575} 576 577static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 578{ 579 struct mlx5e_wqe_frag_info next_frag = {}; 580 struct mlx5e_wqe_frag_info *prev = NULL; 581 int i; 582 583 WARN_ON(rq->xsk_pool); 584 585 next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0]; 586 587 /* Skip first release due to deferred release. */ 588 next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 589 590 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 591 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 592 struct mlx5e_wqe_frag_info *frag = 593 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 594 int f; 595 596 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 597 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 598 /* Pages are assigned at runtime. */ 599 next_frag.frag_page++; 600 next_frag.offset = 0; 601 if (prev) 602 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 603 } 604 *frag = next_frag; 605 606 /* prepare next */ 607 next_frag.offset += frag_info[f].frag_stride; 608 prev = frag; 609 } 610 } 611 612 if (prev) 613 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 614} 615 616static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq) 617{ 618 int i; 619 620 /* Assumptions used by XSK batched allocator. */ 621 WARN_ON(rq->wqe.info.num_frags != 1); 622 WARN_ON(rq->wqe.info.log_num_frags != 0); 623 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); 624 625 /* Considering the above assumptions a fragment maps to a single 626 * xsk_buff. 627 */ 628 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 629 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i]; 630 631 /* Skip first release due to deferred release as WQES are 632 * not allocated yet. 633 */ 634 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 635 } 636} 637 638static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node) 639{ 640 int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 641 int len = wq_sz << rq->wqe.info.log_num_frags; 642 struct mlx5e_wqe_frag_info *frags; 643 union mlx5e_alloc_units *aus; 644 int aus_sz; 645 646 if (rq->xsk_pool) 647 aus_sz = sizeof(*aus->xsk_buffs); 648 else 649 aus_sz = sizeof(*aus->frag_pages); 650 651 aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node); 652 if (!aus) 653 return -ENOMEM; 654 655 frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node); 656 if (!frags) { 657 kvfree(aus); 658 return -ENOMEM; 659 } 660 661 rq->wqe.alloc_units = aus; 662 rq->wqe.frags = frags; 663 664 if (rq->xsk_pool) 665 mlx5e_init_xsk_buffs(rq); 666 else 667 mlx5e_init_frags_partition(rq); 668 669 return 0; 670} 671 672static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq) 673{ 674 kvfree(rq->wqe.frags); 675 kvfree(rq->wqe.alloc_units); 676} 677 678static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 679{ 680 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 681 682 mlx5e_reporter_rq_cqe_err(rq); 683} 684 685static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) 686{ 687 struct mlx5e_rq *rq = container_of(timeout_work, 688 struct mlx5e_rq, 689 rx_timeout_work); 690 691 mlx5e_reporter_rx_timeout(rq); 692} 693 694static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 695{ 696 rq->wqe_overflow.page = alloc_page(GFP_KERNEL); 697 if (!rq->wqe_overflow.page) 698 return -ENOMEM; 699 700 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, 701 PAGE_SIZE, rq->buff.map_dir); 702 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { 703 __free_page(rq->wqe_overflow.page); 704 return -ENOMEM; 705 } 706 return 0; 707} 708 709static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 710{ 711 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE, 712 rq->buff.map_dir); 713 __free_page(rq->wqe_overflow.page); 714} 715 716static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 717 u32 xdp_frag_size, struct mlx5e_rq *rq) 718{ 719 struct mlx5_core_dev *mdev = c->mdev; 720 int err; 721 722 rq->wq_type = params->rq_wq_type; 723 rq->pdev = c->pdev; 724 rq->netdev = c->netdev; 725 rq->priv = c->priv; 726 rq->hwtstamp_config = &c->priv->hwtstamp_config; 727 rq->clock = mdev->clock; 728 rq->icosq = &c->icosq; 729 rq->ix = c->ix; 730 rq->channel = c; 731 rq->mdev = mdev; 732 rq->hw_mtu = 733 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en; 734 rq->xdpsq = &c->rq_xdpsq; 735 rq->stats = &c->priv->channel_stats[c->ix]->rq; 736 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 737 err = mlx5e_rq_set_handlers(rq, params, NULL); 738 if (err) 739 return err; 740 741 return __xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id, 742 xdp_frag_size); 743} 744 745static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, u16 hd_per_wq, 746 int node) 747{ 748 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 749 750 shampo->hd_per_wq = hd_per_wq; 751 752 shampo->bitmap = bitmap_zalloc_node(hd_per_wq, GFP_KERNEL, node); 753 shampo->pages = kvzalloc_node(array_size(hd_per_wq, 754 sizeof(*shampo->pages)), 755 GFP_KERNEL, node); 756 if (!shampo->bitmap || !shampo->pages) 757 goto err_nomem; 758 759 return 0; 760 761err_nomem: 762 kvfree(shampo->pages); 763 bitmap_free(shampo->bitmap); 764 765 return -ENOMEM; 766} 767 768static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) 769{ 770 kvfree(rq->mpwqe.shampo->pages); 771 bitmap_free(rq->mpwqe.shampo->bitmap); 772} 773 774static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 775 struct mlx5e_params *params, 776 struct mlx5e_rq_param *rqp, 777 struct mlx5e_rq *rq, 778 u32 *pool_size, 779 int node) 780{ 781 void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq); 782 u8 log_hd_per_page, log_hd_entry_size; 783 u16 hd_per_wq, hd_per_wqe; 784 u32 hd_pool_size; 785 int wq_size; 786 int err; 787 788 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 789 return 0; 790 791 rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo), 792 GFP_KERNEL, node); 793 if (!rq->mpwqe.shampo) 794 return -ENOMEM; 795 796 /* split headers data structures */ 797 hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); 798 err = mlx5e_rq_shampo_hd_info_alloc(rq, hd_per_wq, node); 799 if (err) 800 goto err_shampo_hd_info_alloc; 801 802 err = mlx5e_create_rq_hd_umr_mkey(mdev, hd_per_wq, 803 &rq->mpwqe.shampo->mkey_be); 804 if (err) 805 goto err_umr_mkey; 806 807 hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rqp); 808 wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); 809 810 BUILD_BUG_ON(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE > PAGE_SHIFT); 811 if (hd_per_wqe >= MLX5E_SHAMPO_WQ_HEADER_PER_PAGE) { 812 log_hd_per_page = MLX5E_SHAMPO_LOG_WQ_HEADER_PER_PAGE; 813 log_hd_entry_size = MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; 814 } else { 815 log_hd_per_page = order_base_2(hd_per_wqe); 816 log_hd_entry_size = order_base_2(PAGE_SIZE / hd_per_wqe); 817 } 818 819 rq->mpwqe.shampo->hd_per_wqe = hd_per_wqe; 820 rq->mpwqe.shampo->hd_per_page = BIT(log_hd_per_page); 821 rq->mpwqe.shampo->log_hd_per_page = log_hd_per_page; 822 rq->mpwqe.shampo->log_hd_entry_size = log_hd_entry_size; 823 824 hd_pool_size = (hd_per_wqe * wq_size) >> log_hd_per_page; 825 826 if (netif_rxq_has_unreadable_mp(rq->netdev, rq->ix)) { 827 /* Separate page pool for shampo headers */ 828 struct page_pool_params pp_params = { }; 829 830 pp_params.order = 0; 831 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 832 pp_params.pool_size = hd_pool_size; 833 pp_params.nid = node; 834 pp_params.dev = rq->pdev; 835 pp_params.napi = rq->cq.napi; 836 pp_params.netdev = rq->netdev; 837 pp_params.dma_dir = rq->buff.map_dir; 838 pp_params.max_len = PAGE_SIZE; 839 840 rq->hd_page_pool = page_pool_create(&pp_params); 841 if (IS_ERR(rq->hd_page_pool)) { 842 err = PTR_ERR(rq->hd_page_pool); 843 rq->hd_page_pool = NULL; 844 goto err_hds_page_pool; 845 } 846 } else { 847 /* Common page pool, reserve space for headers. */ 848 *pool_size += hd_pool_size; 849 rq->hd_page_pool = NULL; 850 } 851 852 /* gro only data structures */ 853 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 854 if (!rq->hw_gro_data) { 855 err = -ENOMEM; 856 goto err_hw_gro_data; 857 } 858 859 return 0; 860 861err_hw_gro_data: 862 page_pool_destroy(rq->hd_page_pool); 863err_hds_page_pool: 864 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 865err_umr_mkey: 866 mlx5e_rq_shampo_hd_info_free(rq); 867err_shampo_hd_info_alloc: 868 kvfree(rq->mpwqe.shampo); 869 return err; 870} 871 872static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 873{ 874 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 875 return; 876 877 kvfree(rq->hw_gro_data); 878 if (rq->hd_page_pool != rq->page_pool) 879 page_pool_destroy(rq->hd_page_pool); 880 mlx5e_rq_shampo_hd_info_free(rq); 881 mlx5_core_destroy_mkey(rq->mdev, 882 be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 883 kvfree(rq->mpwqe.shampo); 884} 885 886static int mlx5e_alloc_rq(struct mlx5e_params *params, 887 struct mlx5e_xsk_param *xsk, 888 struct mlx5e_rq_param *rqp, 889 int node, struct mlx5e_rq *rq) 890{ 891 struct mlx5_core_dev *mdev = rq->mdev; 892 void *rqc = rqp->rqc; 893 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 894 u32 pool_size; 895 int wq_sz; 896 int err; 897 int i; 898 899 rqp->wq.db_numa_node = node; 900 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 901 INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work); 902 903 if (params->xdp_prog) 904 bpf_prog_inc(params->xdp_prog); 905 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); 906 907 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 908 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); 909 pool_size = 1 << params->log_rq_mtu_frames; 910 911 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 912 913 switch (rq->wq_type) { 914 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 915 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 916 &rq->wq_ctrl); 917 if (err) 918 goto err_rq_xdp_prog; 919 920 err = mlx5e_alloc_mpwqe_rq_drop_page(rq); 921 if (err) 922 goto err_rq_wq_destroy; 923 924 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 925 926 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 927 928 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); 929 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); 930 rq->mpwqe.pages_per_wqe = 931 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, 932 rq->mpwqe.umr_mode); 933 rq->mpwqe.umr_wqebbs = 934 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, 935 rq->mpwqe.umr_mode); 936 rq->mpwqe.mtts_per_wqe = 937 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, 938 rq->mpwqe.umr_mode); 939 940 pool_size = rq->mpwqe.pages_per_wqe << 941 mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); 942 943 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog) 944 pool_size *= 2; /* additional page per packet for the linear part */ 945 946 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 947 rq->mpwqe.num_strides = 948 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 949 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); 950 951 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 952 953 err = mlx5e_create_rq_umr_mkey(mdev, rq); 954 if (err) 955 goto err_rq_drop_page; 956 957 err = mlx5e_rq_alloc_mpwqe_info(rq, node); 958 if (err) 959 goto err_rq_mkey; 960 961 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); 962 if (err) 963 goto err_free_mpwqe_info; 964 965 break; 966 default: /* MLX5_WQ_TYPE_CYCLIC */ 967 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 968 &rq->wq_ctrl); 969 if (err) 970 goto err_rq_xdp_prog; 971 972 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 973 974 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 975 976 rq->wqe.info = rqp->frags_info; 977 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 978 979 err = mlx5e_init_wqe_alloc_info(rq, node); 980 if (err) 981 goto err_rq_wq_destroy; 982 } 983 984 if (xsk) { 985 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 986 MEM_TYPE_XSK_BUFF_POOL, NULL); 987 if (err) 988 goto err_free_by_rq_type; 989 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq); 990 } else { 991 /* Create a page_pool and register it with rxq */ 992 struct page_pool_params pp_params = { 0 }; 993 994 pp_params.order = 0; 995 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 996 pp_params.pool_size = pool_size; 997 pp_params.nid = node; 998 pp_params.dev = rq->pdev; 999 pp_params.napi = rq->cq.napi; 1000 pp_params.netdev = rq->netdev; 1001 pp_params.dma_dir = rq->buff.map_dir; 1002 pp_params.max_len = PAGE_SIZE; 1003 pp_params.queue_idx = rq->ix; 1004 1005 /* Shampo header data split allow for unreadable netmem */ 1006 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1007 pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1008 1009 /* page_pool can be used even when there is no rq->xdp_prog, 1010 * given page_pool does not handle DMA mapping there is no 1011 * required state to clear. And page_pool gracefully handle 1012 * elevated refcnt. 1013 */ 1014 rq->page_pool = page_pool_create(&pp_params); 1015 if (IS_ERR(rq->page_pool)) { 1016 err = PTR_ERR(rq->page_pool); 1017 rq->page_pool = NULL; 1018 goto err_free_by_rq_type; 1019 } 1020 if (!rq->hd_page_pool) 1021 rq->hd_page_pool = rq->page_pool; 1022 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1023 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1024 MEM_TYPE_PAGE_POOL, rq->page_pool); 1025 if (err) 1026 goto err_destroy_page_pool; 1027 } 1028 } 1029 1030 for (i = 0; i < wq_sz; i++) { 1031 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1032 struct mlx5e_rx_wqe_ll *wqe = 1033 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 1034 u32 byte_count = 1035 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 1036 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << 1037 rq->mpwqe.page_shift; 1038 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 1039 0 : rq->buff.headroom; 1040 1041 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); 1042 wqe->data[0].byte_count = cpu_to_be32(byte_count); 1043 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; 1044 } else { 1045 struct mlx5e_rx_wqe_cyc *wqe = 1046 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 1047 int f; 1048 1049 for (f = 0; f < rq->wqe.info.num_frags; f++) { 1050 u32 frag_size = rq->wqe.info.arr[f].frag_size | 1051 MLX5_HW_START_PADDING; 1052 1053 wqe->data[f].byte_count = cpu_to_be32(frag_size); 1054 wqe->data[f].lkey = rq->mkey_be; 1055 } 1056 /* check if num_frags is not a pow of two */ 1057 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 1058 wqe->data[f].byte_count = 0; 1059 wqe->data[f].lkey = params->terminate_lkey_be; 1060 wqe->data[f].addr = 0; 1061 } 1062 } 1063 } 1064 1065 return 0; 1066 1067err_destroy_page_pool: 1068 page_pool_destroy(rq->page_pool); 1069err_free_by_rq_type: 1070 switch (rq->wq_type) { 1071 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1072 mlx5e_rq_free_shampo(rq); 1073err_free_mpwqe_info: 1074 kvfree(rq->mpwqe.info); 1075err_rq_mkey: 1076 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1077err_rq_drop_page: 1078 mlx5e_free_mpwqe_rq_drop_page(rq); 1079 break; 1080 default: /* MLX5_WQ_TYPE_CYCLIC */ 1081 mlx5e_free_wqe_alloc_info(rq); 1082 } 1083err_rq_wq_destroy: 1084 mlx5_wq_destroy(&rq->wq_ctrl); 1085err_rq_xdp_prog: 1086 if (params->xdp_prog) 1087 bpf_prog_put(params->xdp_prog); 1088 1089 return err; 1090} 1091 1092static void mlx5e_free_rq(struct mlx5e_rq *rq) 1093{ 1094 kvfree(rq->dim); 1095 page_pool_destroy(rq->page_pool); 1096 1097 switch (rq->wq_type) { 1098 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1099 mlx5e_rq_free_shampo(rq); 1100 kvfree(rq->mpwqe.info); 1101 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1102 mlx5e_free_mpwqe_rq_drop_page(rq); 1103 break; 1104 default: /* MLX5_WQ_TYPE_CYCLIC */ 1105 mlx5e_free_wqe_alloc_info(rq); 1106 } 1107 1108 mlx5_wq_destroy(&rq->wq_ctrl); 1109 1110 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1111 struct bpf_prog *old_prog; 1112 1113 old_prog = rcu_dereference_protected(rq->xdp_prog, 1114 lockdep_is_held(&rq->priv->state_lock)); 1115 if (old_prog) 1116 bpf_prog_put(old_prog); 1117 } 1118 xdp_rxq_info_unreg(&rq->xdp_rxq); 1119} 1120 1121int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) 1122{ 1123 struct mlx5_core_dev *mdev = rq->mdev; 1124 u8 ts_format; 1125 void *in; 1126 void *rqc; 1127 void *wq; 1128 int inlen; 1129 int err; 1130 1131 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 1132 sizeof(u64) * rq->wq_ctrl.buf.npages; 1133 in = kvzalloc(inlen, GFP_KERNEL); 1134 if (!in) 1135 return -ENOMEM; 1136 1137 ts_format = mlx5_is_real_time_rq(mdev) ? 1138 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1139 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1140 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 1141 wq = MLX5_ADDR_OF(rqc, rqc, wq); 1142 1143 memcpy(rqc, param->rqc, sizeof(param->rqc)); 1144 1145 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 1146 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 1147 MLX5_SET(rqc, rqc, ts_format, ts_format); 1148 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 1149 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 1150 MLX5_ADAPTER_PAGE_SHIFT); 1151 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 1152 1153 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1154 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 1155 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 1156 MLX5_SET(wq, wq, headers_mkey, 1157 be32_to_cpu(rq->mpwqe.shampo->mkey_be)); 1158 } 1159 1160 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 1161 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1162 1163 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 1164 1165 kvfree(in); 1166 1167 return err; 1168} 1169 1170static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 1171{ 1172 struct mlx5_core_dev *mdev = rq->mdev; 1173 1174 void *in; 1175 void *rqc; 1176 int inlen; 1177 int err; 1178 1179 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1180 in = kvzalloc(inlen, GFP_KERNEL); 1181 if (!in) 1182 return -ENOMEM; 1183 1184 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 1185 mlx5e_rqwq_reset(rq); 1186 1187 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1188 1189 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 1190 MLX5_SET(rqc, rqc, state, next_state); 1191 1192 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1193 1194 kvfree(in); 1195 1196 return err; 1197} 1198 1199static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq) 1200{ 1201 struct mlx5_cqwq *cqwq = &rq->cq.wq; 1202 struct mlx5_cqe64 *cqe; 1203 1204 if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) { 1205 while ((cqe = mlx5_cqwq_get_cqe_enhanced_comp(cqwq))) 1206 mlx5_cqwq_pop(cqwq); 1207 } else { 1208 while ((cqe = mlx5_cqwq_get_cqe(cqwq))) 1209 mlx5_cqwq_pop(cqwq); 1210 } 1211 1212 mlx5_cqwq_update_db_record(cqwq); 1213} 1214 1215int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) 1216{ 1217 struct net_device *dev = rq->netdev; 1218 int err; 1219 1220 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 1221 if (err) { 1222 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 1223 return err; 1224 } 1225 1226 mlx5e_free_rx_descs(rq); 1227 mlx5e_flush_rq_cq(rq); 1228 1229 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1230 if (err) { 1231 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 1232 return err; 1233 } 1234 1235 return 0; 1236} 1237 1238static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 1239{ 1240 struct mlx5_core_dev *mdev = rq->mdev; 1241 void *in; 1242 void *rqc; 1243 int inlen; 1244 int err; 1245 1246 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1247 in = kvzalloc(inlen, GFP_KERNEL); 1248 if (!in) 1249 return -ENOMEM; 1250 1251 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1252 1253 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1254 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1255 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 1256 MLX5_SET(rqc, rqc, vsd, vsd); 1257 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1258 1259 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1260 1261 kvfree(in); 1262 1263 return err; 1264} 1265 1266void mlx5e_destroy_rq(struct mlx5e_rq *rq) 1267{ 1268 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 1269} 1270 1271int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 1272{ 1273 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 1274 1275 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 1276 1277 do { 1278 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 1279 return 0; 1280 1281 msleep(20); 1282 } while (time_before(jiffies, exp_time)); 1283 1284 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 1285 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 1286 1287 queue_work(rq->priv->wq, &rq->rx_timeout_work); 1288 1289 return -ETIMEDOUT; 1290} 1291 1292void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq) 1293{ 1294 struct mlx5_wq_ll *wq; 1295 u16 head; 1296 int i; 1297 1298 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 1299 return; 1300 1301 wq = &rq->mpwqe.wq; 1302 head = wq->head; 1303 1304 /* Release WQEs that are in missing state: they have been 1305 * popped from the list after completion but were not freed 1306 * due to deferred release. 1307 * Also free the linked-list reserved entry, hence the "+ 1". 1308 */ 1309 for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) { 1310 rq->dealloc_wqe(rq, head); 1311 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1312 } 1313 1314 rq->mpwqe.actual_wq_head = wq->head; 1315 rq->mpwqe.umr_in_progress = 0; 1316 rq->mpwqe.umr_completed = 0; 1317 1318 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1319 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 1320 u16 len; 1321 1322 len = (shampo->pi - shampo->ci) & shampo->hd_per_wq; 1323 mlx5e_shampo_fill_umr(rq, len); 1324 } 1325} 1326 1327void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 1328{ 1329 __be16 wqe_ix_be; 1330 u16 wqe_ix; 1331 1332 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1333 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 1334 1335 mlx5e_free_rx_missing_descs(rq); 1336 1337 while (!mlx5_wq_ll_is_empty(wq)) { 1338 struct mlx5e_rx_wqe_ll *wqe; 1339 1340 wqe_ix_be = *wq->tail_next; 1341 wqe_ix = be16_to_cpu(wqe_ix_be); 1342 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 1343 rq->dealloc_wqe(rq, wqe_ix); 1344 mlx5_wq_ll_pop(wq, wqe_ix_be, 1345 &wqe->next.next_wqe_index); 1346 } 1347 1348 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1349 mlx5e_shampo_dealloc_hd(rq); 1350 } else { 1351 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1352 u16 missing = mlx5_wq_cyc_missing(wq); 1353 u16 head = mlx5_wq_cyc_get_head(wq); 1354 1355 while (!mlx5_wq_cyc_is_empty(wq)) { 1356 wqe_ix = mlx5_wq_cyc_get_tail(wq); 1357 rq->dealloc_wqe(rq, wqe_ix); 1358 mlx5_wq_cyc_pop(wq); 1359 } 1360 /* Missing slots might also contain unreleased pages due to 1361 * deferred release. 1362 */ 1363 while (missing--) { 1364 wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++); 1365 rq->dealloc_wqe(rq, wqe_ix); 1366 } 1367 } 1368 1369} 1370 1371int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, 1372 struct mlx5e_xsk_param *xsk, int node, u16 q_counter, 1373 struct mlx5e_rq *rq) 1374{ 1375 struct mlx5_core_dev *mdev = rq->mdev; 1376 int err; 1377 1378 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1379 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); 1380 1381 err = mlx5e_alloc_rq(params, xsk, param, node, rq); 1382 if (err) 1383 return err; 1384 1385 err = mlx5e_create_rq(rq, param, q_counter); 1386 if (err) 1387 goto err_free_rq; 1388 1389 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1390 if (err) 1391 goto err_destroy_rq; 1392 1393 if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) 1394 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state); 1395 1396 if (rq->channel && !params->rx_dim_enabled) { 1397 rq->channel->rx_cq_moder = params->rx_cq_moderation; 1398 } else if (rq->channel) { 1399 u8 cq_period_mode; 1400 1401 cq_period_mode = params->rx_moder_use_cqe_mode ? 1402 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1403 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1404 mlx5e_reset_rx_moderation(&rq->channel->rx_cq_moder, cq_period_mode, 1405 params->rx_dim_enabled); 1406 1407 err = mlx5e_dim_rx_change(rq, params->rx_dim_enabled); 1408 if (err) 1409 goto err_destroy_rq; 1410 } 1411 1412 /* We disable csum_complete when XDP is enabled since 1413 * XDP programs might manipulate packets which will render 1414 * skb->checksum incorrect. 1415 */ 1416 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog) 1417 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state); 1418 1419 /* For CQE compression on striding RQ, use stride index provided by 1420 * HW if capability is supported. 1421 */ 1422 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) && 1423 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) 1424 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); 1425 1426 /* For enhanced CQE compression packet processing. decompress 1427 * session according to the enhanced layout. 1428 */ 1429 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && 1430 MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) 1431 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); 1432 1433 return 0; 1434 1435err_destroy_rq: 1436 mlx5e_destroy_rq(rq); 1437err_free_rq: 1438 mlx5e_free_rq(rq); 1439 1440 return err; 1441} 1442 1443void mlx5e_activate_rq(struct mlx5e_rq *rq) 1444{ 1445 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1446} 1447 1448void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 1449{ 1450 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1451 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 1452} 1453 1454void mlx5e_close_rq(struct mlx5e_rq *rq) 1455{ 1456 if (rq->dim) 1457 cancel_work_sync(&rq->dim->work); 1458 cancel_work_sync(&rq->recover_work); 1459 cancel_work_sync(&rq->rx_timeout_work); 1460 mlx5e_destroy_rq(rq); 1461 mlx5e_free_rx_descs(rq); 1462 mlx5e_free_rq(rq); 1463} 1464 1465u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, 1466 struct mlx5e_priv *priv, 1467 const struct mlx5e_profile *profile, 1468 u8 lag_port, u8 tc) 1469{ 1470 if (profile->get_tisn) 1471 return profile->get_tisn(mdev, priv, lag_port, tc); 1472 1473 return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; 1474} 1475 1476static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 1477{ 1478 kvfree(sq->db.xdpi_fifo.xi); 1479 kvfree(sq->db.wqe_info); 1480} 1481 1482static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 1483{ 1484 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 1485 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1486 int entries; 1487 size_t size; 1488 1489 /* upper bound for maximum num of entries of all xmit_modes. */ 1490 entries = roundup_pow_of_two(wq_sz * MLX5_SEND_WQEBB_NUM_DS * 1491 MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO); 1492 1493 size = array_size(sizeof(*xdpi_fifo->xi), entries); 1494 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa); 1495 if (!xdpi_fifo->xi) 1496 return -ENOMEM; 1497 1498 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 1499 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 1500 xdpi_fifo->mask = entries - 1; 1501 1502 return 0; 1503} 1504 1505static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 1506{ 1507 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1508 size_t size; 1509 int err; 1510 1511 size = array_size(sizeof(*sq->db.wqe_info), wq_sz); 1512 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1513 if (!sq->db.wqe_info) 1514 return -ENOMEM; 1515 1516 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 1517 if (err) { 1518 mlx5e_free_xdpsq_db(sq); 1519 return err; 1520 } 1521 1522 return 0; 1523} 1524 1525static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 1526 struct mlx5e_params *params, 1527 struct xsk_buff_pool *xsk_pool, 1528 struct mlx5e_sq_param *param, 1529 struct mlx5e_xdpsq *sq, 1530 bool is_redirect) 1531{ 1532 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1533 struct mlx5_core_dev *mdev = c->mdev; 1534 struct mlx5_wq_cyc *wq = &sq->wq; 1535 int err; 1536 1537 sq->pdev = c->pdev; 1538 sq->mkey_be = c->mkey_be; 1539 sq->channel = c; 1540 sq->uar_map = c->bfreg->map; 1541 sq->min_inline_mode = params->tx_min_inline_mode; 1542 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN; 1543 sq->xsk_pool = xsk_pool; 1544 1545 sq->stats = sq->xsk_pool ? 1546 &c->priv->channel_stats[c->ix]->xsksq : 1547 is_redirect ? 1548 &c->priv->channel_stats[c->ix]->xdpsq : 1549 &c->priv->channel_stats[c->ix]->rq_xdpsq; 1550 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : 1551 mlx5e_stop_room_for_max_wqe(mdev); 1552 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1553 1554 param->wq.db_numa_node = cpu_to_node(c->cpu); 1555 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1556 if (err) 1557 return err; 1558 wq->db = &wq->db[MLX5_SND_DBR]; 1559 1560 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 1561 if (err) 1562 goto err_sq_wq_destroy; 1563 1564 return 0; 1565 1566err_sq_wq_destroy: 1567 mlx5_wq_destroy(&sq->wq_ctrl); 1568 1569 return err; 1570} 1571 1572static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1573{ 1574 mlx5e_free_xdpsq_db(sq); 1575 mlx5_wq_destroy(&sq->wq_ctrl); 1576} 1577 1578static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1579{ 1580 kvfree(sq->db.wqe_info); 1581} 1582 1583static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1584{ 1585 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1586 size_t size; 1587 1588 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1589 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1590 if (!sq->db.wqe_info) 1591 return -ENOMEM; 1592 1593 return 0; 1594} 1595 1596static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1597{ 1598 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1599 recover_work); 1600 1601 mlx5e_reporter_icosq_cqe_err(sq); 1602} 1603 1604static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work) 1605{ 1606 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1607 recover_work); 1608 1609 /* Not implemented yet. */ 1610 1611 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n"); 1612} 1613 1614static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1615 struct mlx5e_sq_param *param, 1616 struct mlx5e_icosq *sq, 1617 work_func_t recover_work_func) 1618{ 1619 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1620 struct mlx5_core_dev *mdev = c->mdev; 1621 struct mlx5_wq_cyc *wq = &sq->wq; 1622 int err; 1623 1624 sq->channel = c; 1625 sq->uar_map = c->bfreg->map; 1626 sq->reserved_room = param->stop_room; 1627 1628 param->wq.db_numa_node = cpu_to_node(c->cpu); 1629 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1630 if (err) 1631 return err; 1632 wq->db = &wq->db[MLX5_SND_DBR]; 1633 1634 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1635 if (err) 1636 goto err_sq_wq_destroy; 1637 1638 INIT_WORK(&sq->recover_work, recover_work_func); 1639 1640 return 0; 1641 1642err_sq_wq_destroy: 1643 mlx5_wq_destroy(&sq->wq_ctrl); 1644 1645 return err; 1646} 1647 1648static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1649{ 1650 mlx5e_free_icosq_db(sq); 1651 mlx5_wq_destroy(&sq->wq_ctrl); 1652} 1653 1654void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1655{ 1656 kvfree(sq->db.wqe_info); 1657 kvfree(sq->db.skb_fifo.fifo); 1658 kvfree(sq->db.dma_fifo); 1659} 1660 1661int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1662{ 1663 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1664 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1665 1666 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1667 sizeof(*sq->db.dma_fifo)), 1668 GFP_KERNEL, numa); 1669 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz, 1670 sizeof(*sq->db.skb_fifo.fifo)), 1671 GFP_KERNEL, numa); 1672 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1673 sizeof(*sq->db.wqe_info)), 1674 GFP_KERNEL, numa); 1675 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) { 1676 mlx5e_free_txqsq_db(sq); 1677 return -ENOMEM; 1678 } 1679 1680 sq->dma_fifo_mask = df_sz - 1; 1681 1682 sq->db.skb_fifo.pc = &sq->skb_fifo_pc; 1683 sq->db.skb_fifo.cc = &sq->skb_fifo_cc; 1684 sq->db.skb_fifo.mask = df_sz - 1; 1685 1686 return 0; 1687} 1688 1689static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1690 int txq_ix, 1691 struct mlx5e_params *params, 1692 struct mlx5e_sq_param *param, 1693 struct mlx5e_txqsq *sq, 1694 int tc) 1695{ 1696 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1697 struct mlx5_core_dev *mdev = c->mdev; 1698 struct mlx5_wq_cyc *wq = &sq->wq; 1699 int err; 1700 1701 sq->pdev = c->pdev; 1702 sq->clock = mdev->clock; 1703 sq->mkey_be = c->mkey_be; 1704 sq->netdev = c->netdev; 1705 sq->mdev = c->mdev; 1706 sq->channel = c; 1707 sq->priv = c->priv; 1708 sq->ch_ix = c->ix; 1709 sq->txq_ix = txq_ix; 1710 sq->uar_map = c->bfreg->map; 1711 sq->min_inline_mode = params->tx_min_inline_mode; 1712 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1713 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1714 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1715 if (mlx5_ipsec_device_caps(c->priv->mdev)) 1716 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1717 if (param->is_mpw) 1718 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state); 1719 sq->stop_room = param->stop_room; 1720 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev); 1721 1722 param->wq.db_numa_node = cpu_to_node(c->cpu); 1723 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1724 if (err) 1725 return err; 1726 wq->db = &wq->db[MLX5_SND_DBR]; 1727 1728 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1729 if (err) 1730 goto err_sq_wq_destroy; 1731 1732 return 0; 1733 1734err_sq_wq_destroy: 1735 mlx5_wq_destroy(&sq->wq_ctrl); 1736 1737 return err; 1738} 1739 1740void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1741{ 1742 kvfree(sq->dim); 1743 mlx5e_free_txqsq_db(sq); 1744 mlx5_wq_destroy(&sq->wq_ctrl); 1745} 1746 1747static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1748 struct mlx5e_sq_param *param, 1749 struct mlx5e_create_sq_param *csp, 1750 u32 *sqn) 1751{ 1752 u8 ts_format; 1753 void *in; 1754 void *sqc; 1755 void *wq; 1756 int inlen; 1757 int err; 1758 1759 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1760 sizeof(u64) * csp->wq_ctrl->buf.npages; 1761 in = kvzalloc(inlen, GFP_KERNEL); 1762 if (!in) 1763 return -ENOMEM; 1764 1765 ts_format = mlx5_is_real_time_sq(mdev) ? 1766 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1767 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1768 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1769 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1770 1771 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1772 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1773 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1774 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1775 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn); 1776 MLX5_SET(sqc, sqc, ts_format, ts_format); 1777 1778 1779 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1780 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1781 1782 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1783 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1784 1785 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1786 MLX5_SET(wq, wq, uar_page, csp->uar_page); 1787 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1788 MLX5_ADAPTER_PAGE_SHIFT); 1789 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1790 1791 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1792 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1793 1794 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1795 1796 kvfree(in); 1797 1798 return err; 1799} 1800 1801int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1802 struct mlx5e_modify_sq_param *p) 1803{ 1804 u64 bitmask = 0; 1805 void *in; 1806 void *sqc; 1807 int inlen; 1808 int err; 1809 1810 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1811 in = kvzalloc(inlen, GFP_KERNEL); 1812 if (!in) 1813 return -ENOMEM; 1814 1815 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1816 1817 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1818 MLX5_SET(sqc, sqc, state, p->next_state); 1819 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1820 bitmask |= 1; 1821 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1822 } 1823 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { 1824 bitmask |= 1 << 2; 1825 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); 1826 } 1827 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); 1828 1829 err = mlx5_core_modify_sq(mdev, sqn, in); 1830 1831 kvfree(in); 1832 1833 return err; 1834} 1835 1836static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1837{ 1838 mlx5_core_destroy_sq(mdev, sqn); 1839} 1840 1841int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1842 struct mlx5e_sq_param *param, 1843 struct mlx5e_create_sq_param *csp, 1844 u16 qos_queue_group_id, 1845 u32 *sqn) 1846{ 1847 struct mlx5e_modify_sq_param msp = {0}; 1848 int err; 1849 1850 err = mlx5e_create_sq(mdev, param, csp, sqn); 1851 if (err) 1852 return err; 1853 1854 msp.curr_state = MLX5_SQC_STATE_RST; 1855 msp.next_state = MLX5_SQC_STATE_RDY; 1856 if (qos_queue_group_id) { 1857 msp.qos_update = true; 1858 msp.qos_queue_group_id = qos_queue_group_id; 1859 } 1860 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1861 if (err) 1862 mlx5e_destroy_sq(mdev, *sqn); 1863 1864 return err; 1865} 1866 1867static int mlx5e_set_sq_maxrate(struct net_device *dev, 1868 struct mlx5e_txqsq *sq, u32 rate); 1869 1870int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, 1871 struct mlx5e_params *params, struct mlx5e_sq_param *param, 1872 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, 1873 struct mlx5e_sq_stats *sq_stats) 1874{ 1875 struct mlx5e_create_sq_param csp = {}; 1876 u32 tx_rate; 1877 int err; 1878 1879 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1880 if (err) 1881 return err; 1882 1883 sq->stats = sq_stats; 1884 1885 csp.tisn = tisn; 1886 csp.tis_lst_sz = 1; 1887 csp.cqn = sq->cq.mcq.cqn; 1888 csp.wq_ctrl = &sq->wq_ctrl; 1889 csp.min_inline_mode = sq->min_inline_mode; 1890 csp.uar_page = c->bfreg->index; 1891 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); 1892 if (err) 1893 goto err_free_txqsq; 1894 1895 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1896 if (tx_rate) 1897 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1898 1899 if (sq->channel && !params->tx_dim_enabled) { 1900 sq->channel->tx_cq_moder = params->tx_cq_moderation; 1901 } else if (sq->channel) { 1902 u8 cq_period_mode; 1903 1904 cq_period_mode = params->tx_moder_use_cqe_mode ? 1905 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1906 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1907 mlx5e_reset_tx_moderation(&sq->channel->tx_cq_moder, 1908 cq_period_mode, 1909 params->tx_dim_enabled); 1910 1911 err = mlx5e_dim_tx_change(sq, params->tx_dim_enabled); 1912 if (err) 1913 goto err_destroy_sq; 1914 } 1915 1916 return 0; 1917 1918err_destroy_sq: 1919 mlx5e_destroy_sq(c->mdev, sq->sqn); 1920err_free_txqsq: 1921 mlx5e_free_txqsq(sq); 1922 1923 return err; 1924} 1925 1926void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1927{ 1928 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); 1929 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1930 netdev_tx_reset_queue(sq->txq); 1931 netif_tx_start_queue(sq->txq); 1932 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); 1933} 1934 1935void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1936{ 1937 __netif_tx_lock_bh(txq); 1938 netif_tx_stop_queue(txq); 1939 __netif_tx_unlock_bh(txq); 1940} 1941 1942void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1943{ 1944 struct mlx5_wq_cyc *wq = &sq->wq; 1945 1946 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL); 1947 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1948 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ 1949 1950 mlx5e_tx_disable_queue(sq->txq); 1951 1952 /* last doorbell out, godspeed .. */ 1953 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1954 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1955 struct mlx5e_tx_wqe *nop; 1956 1957 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1958 .num_wqebbs = 1, 1959 }; 1960 1961 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1962 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1963 } 1964} 1965 1966void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1967{ 1968 struct mlx5_core_dev *mdev = sq->mdev; 1969 struct mlx5_rate_limit rl = {0}; 1970 1971 if (sq->dim) 1972 cancel_work_sync(&sq->dim->work); 1973 cancel_work_sync(&sq->recover_work); 1974 mlx5e_destroy_sq(mdev, sq->sqn); 1975 if (sq->rate_limit) { 1976 rl.rate = sq->rate_limit; 1977 mlx5_rl_remove_rate(mdev, &rl); 1978 } 1979 mlx5e_free_txqsq_descs(sq); 1980 mlx5e_free_txqsq(sq); 1981} 1982 1983void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1984{ 1985 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1986 recover_work); 1987 1988 mlx5e_reporter_tx_err_cqe(sq); 1989} 1990 1991static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 1992{ 1993 return (struct dim_cq_moder) { 1994 .cq_period_mode = cq_period_mode, 1995 .pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS, 1996 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 1997 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE : 1998 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC, 1999 }; 2000} 2001 2002bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 2003 bool dim_enabled) 2004{ 2005 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 2006 2007 if (dim_enabled) 2008 *cq_moder = net_dim_get_def_tx_moderation(cq_period_mode); 2009 else 2010 *cq_moder = mlx5e_get_def_tx_moderation(cq_period_mode); 2011 2012 return reset_needed; 2013} 2014 2015bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 2016 bool dim_enabled, bool keep_dim_state) 2017{ 2018 bool reset = false; 2019 int i, tc; 2020 2021 for (i = 0; i < chs->num; i++) { 2022 for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) { 2023 if (keep_dim_state) 2024 dim_enabled = !!chs->c[i]->sq[tc].dim; 2025 2026 reset |= mlx5e_reset_tx_moderation(&chs->c[i]->tx_cq_moder, 2027 cq_period_mode, dim_enabled); 2028 } 2029 } 2030 2031 return reset; 2032} 2033 2034static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 2035 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq, 2036 work_func_t recover_work_func) 2037{ 2038 struct mlx5e_create_sq_param csp = {}; 2039 int err; 2040 2041 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func); 2042 if (err) 2043 return err; 2044 2045 csp.cqn = sq->cq.mcq.cqn; 2046 csp.wq_ctrl = &sq->wq_ctrl; 2047 csp.min_inline_mode = params->tx_min_inline_mode; 2048 csp.uar_page = c->bfreg->index; 2049 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2050 if (err) 2051 goto err_free_icosq; 2052 2053 if (param->is_tls) { 2054 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 2055 if (IS_ERR(sq->ktls_resync)) { 2056 err = PTR_ERR(sq->ktls_resync); 2057 goto err_destroy_icosq; 2058 } 2059 } 2060 return 0; 2061 2062err_destroy_icosq: 2063 mlx5e_destroy_sq(c->mdev, sq->sqn); 2064err_free_icosq: 2065 mlx5e_free_icosq(sq); 2066 2067 return err; 2068} 2069 2070void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 2071{ 2072 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2073} 2074 2075void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 2076{ 2077 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2078 synchronize_net(); /* Sync with NAPI. */ 2079} 2080 2081static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 2082{ 2083 struct mlx5e_channel *c = sq->channel; 2084 2085 if (sq->ktls_resync) 2086 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync); 2087 mlx5e_destroy_sq(c->mdev, sq->sqn); 2088 mlx5e_free_icosq_descs(sq); 2089 mlx5e_free_icosq(sq); 2090} 2091 2092int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 2093 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 2094 struct mlx5e_xdpsq *sq, bool is_redirect) 2095{ 2096 struct mlx5e_create_sq_param csp = {}; 2097 int err; 2098 2099 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect); 2100 if (err) 2101 return err; 2102 2103 csp.tis_lst_sz = 1; 2104 csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2105 c->lag_port, 0); /* tc = 0 */ 2106 csp.cqn = sq->cq.mcq.cqn; 2107 csp.wq_ctrl = &sq->wq_ctrl; 2108 csp.min_inline_mode = sq->min_inline_mode; 2109 csp.uar_page = c->bfreg->index; 2110 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2111 2112 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2113 if (err) 2114 goto err_free_xdpsq; 2115 2116 mlx5e_set_xmit_fp(sq, param->is_mpw); 2117 2118 return 0; 2119 2120err_free_xdpsq: 2121 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2122 mlx5e_free_xdpsq(sq); 2123 2124 return err; 2125} 2126 2127void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 2128{ 2129 struct mlx5e_channel *c = sq->channel; 2130 2131 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2132 synchronize_net(); /* Sync with NAPI. */ 2133 2134 mlx5e_destroy_sq(c->mdev, sq->sqn); 2135 mlx5e_free_xdpsq_descs(sq); 2136 mlx5e_free_xdpsq(sq); 2137} 2138 2139static struct mlx5e_xdpsq *mlx5e_open_xdpredirect_sq(struct mlx5e_channel *c, 2140 struct mlx5e_params *params, 2141 struct mlx5e_channel_param *cparam, 2142 struct mlx5e_create_cq_param *ccp) 2143{ 2144 struct mlx5e_xdpsq *xdpsq; 2145 int err; 2146 2147 xdpsq = kvzalloc_node(sizeof(*xdpsq), GFP_KERNEL, cpu_to_node(c->cpu)); 2148 if (!xdpsq) 2149 return ERR_PTR(-ENOMEM); 2150 2151 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, 2152 &cparam->xdp_sq.cqp, ccp, &xdpsq->cq); 2153 if (err) 2154 goto err_free_xdpsq; 2155 2156 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, xdpsq, true); 2157 if (err) 2158 goto err_close_xdpsq_cq; 2159 2160 return xdpsq; 2161 2162err_close_xdpsq_cq: 2163 mlx5e_close_cq(&xdpsq->cq); 2164err_free_xdpsq: 2165 kvfree(xdpsq); 2166 2167 return ERR_PTR(err); 2168} 2169 2170static void mlx5e_close_xdpredirect_sq(struct mlx5e_xdpsq *xdpsq) 2171{ 2172 mlx5e_close_xdpsq(xdpsq); 2173 mlx5e_close_cq(&xdpsq->cq); 2174 kvfree(xdpsq); 2175} 2176 2177static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 2178 struct net_device *netdev, 2179 struct workqueue_struct *workqueue, 2180 struct mlx5_uars_page *uar, 2181 struct mlx5e_cq_param *param, 2182 struct mlx5e_cq *cq) 2183{ 2184 struct mlx5_core_cq *mcq = &cq->mcq; 2185 int err; 2186 u32 i; 2187 2188 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 2189 &cq->wq_ctrl); 2190 if (err) 2191 return err; 2192 2193 mcq->cqe_sz = 64; 2194 mcq->set_ci_db = cq->wq_ctrl.db.db; 2195 mcq->arm_db = cq->wq_ctrl.db.db + 1; 2196 *mcq->set_ci_db = 0; 2197 mcq->vector = param->eq_ix; 2198 mcq->comp = mlx5e_completion_event; 2199 mcq->event = mlx5e_cq_error_event; 2200 2201 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 2202 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 2203 2204 cqe->op_own = 0xf1; 2205 cqe->validity_iteration_count = 0xff; 2206 } 2207 2208 cq->mdev = mdev; 2209 cq->netdev = netdev; 2210 cq->workqueue = workqueue; 2211 cq->uar = uar; 2212 2213 return 0; 2214} 2215 2216static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, 2217 struct mlx5e_cq_param *param, 2218 struct mlx5e_create_cq_param *ccp, 2219 struct mlx5e_cq *cq) 2220{ 2221 int err; 2222 2223 param->wq.buf_numa_node = ccp->node; 2224 param->wq.db_numa_node = ccp->node; 2225 param->eq_ix = ccp->ix; 2226 2227 err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, 2228 ccp->uar, param, cq); 2229 2230 cq->napi = ccp->napi; 2231 cq->ch_stats = ccp->ch_stats; 2232 2233 return err; 2234} 2235 2236static void mlx5e_free_cq(struct mlx5e_cq *cq) 2237{ 2238 mlx5_wq_destroy(&cq->wq_ctrl); 2239} 2240 2241static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 2242{ 2243 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 2244 struct mlx5_core_dev *mdev = cq->mdev; 2245 struct mlx5_core_cq *mcq = &cq->mcq; 2246 2247 void *in; 2248 void *cqc; 2249 int inlen; 2250 int eqn; 2251 int err; 2252 2253 err = mlx5_comp_eqn_get(mdev, param->eq_ix, &eqn); 2254 if (err) 2255 return err; 2256 2257 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 2258 sizeof(u64) * cq->wq_ctrl.buf.npages; 2259 in = kvzalloc(inlen, GFP_KERNEL); 2260 if (!in) 2261 return -ENOMEM; 2262 2263 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 2264 2265 memcpy(cqc, param->cqc, sizeof(param->cqc)); 2266 2267 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 2268 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 2269 2270 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode)); 2271 2272 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 2273 MLX5_SET(cqc, cqc, uar_page, cq->uar->index); 2274 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 2275 MLX5_ADAPTER_PAGE_SHIFT); 2276 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 2277 2278 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 2279 2280 kvfree(in); 2281 2282 if (err) 2283 return err; 2284 2285 mlx5e_cq_arm(cq); 2286 2287 return 0; 2288} 2289 2290static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 2291{ 2292 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 2293} 2294 2295int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, 2296 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, 2297 struct mlx5e_cq *cq) 2298{ 2299 int err; 2300 2301 err = mlx5e_alloc_cq(mdev, param, ccp, cq); 2302 if (err) 2303 return err; 2304 2305 err = mlx5e_create_cq(cq, param); 2306 if (err) 2307 goto err_free_cq; 2308 2309 if (MLX5_CAP_GEN(mdev, cq_moderation) && 2310 MLX5_CAP_GEN(mdev, cq_period_mode_modify)) 2311 mlx5e_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts, 2312 mlx5e_cq_period_mode(moder.cq_period_mode)); 2313 return 0; 2314 2315err_free_cq: 2316 mlx5e_free_cq(cq); 2317 2318 return err; 2319} 2320 2321void mlx5e_close_cq(struct mlx5e_cq *cq) 2322{ 2323 mlx5e_destroy_cq(cq); 2324 mlx5e_free_cq(cq); 2325} 2326 2327int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2328 u8 cq_period_mode) 2329{ 2330 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2331 void *cqc; 2332 2333 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2334 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2335 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(cq_period_mode)); 2336 MLX5_SET(modify_cq_in, in, 2337 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2338 MLX5_CQ_MODIFY_PERIOD_MODE); 2339 2340 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2341} 2342 2343int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2344 u16 cq_period, u16 cq_max_count, u8 cq_period_mode) 2345{ 2346 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2347 void *cqc; 2348 2349 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2350 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2351 MLX5_SET(cqc, cqc, cq_period, cq_period); 2352 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 2353 MLX5_SET(cqc, cqc, cq_period_mode, cq_period_mode); 2354 MLX5_SET(modify_cq_in, in, 2355 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2356 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE); 2357 2358 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2359} 2360 2361static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 2362 struct mlx5e_params *params, 2363 struct mlx5e_create_cq_param *ccp, 2364 struct mlx5e_channel_param *cparam) 2365{ 2366 int err; 2367 int tc; 2368 2369 for (tc = 0; tc < c->num_tc; tc++) { 2370 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp, 2371 ccp, &c->sq[tc].cq); 2372 if (err) 2373 goto err_close_tx_cqs; 2374 } 2375 2376 return 0; 2377 2378err_close_tx_cqs: 2379 for (tc--; tc >= 0; tc--) 2380 mlx5e_close_cq(&c->sq[tc].cq); 2381 2382 return err; 2383} 2384 2385static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 2386{ 2387 int tc; 2388 2389 for (tc = 0; tc < c->num_tc; tc++) 2390 mlx5e_close_cq(&c->sq[tc].cq); 2391} 2392 2393static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq) 2394{ 2395 int tc; 2396 2397 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 2398 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count) 2399 return tc; 2400 2401 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq); 2402 return -ENOENT; 2403} 2404 2405static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, 2406 u32 *hw_id) 2407{ 2408 int tc; 2409 2410 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { 2411 *hw_id = 0; 2412 return 0; 2413 } 2414 2415 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix); 2416 if (tc < 0) 2417 return tc; 2418 2419 if (tc >= params->mqprio.num_tc) { 2420 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", 2421 tc, params->mqprio.num_tc); 2422 return -EINVAL; 2423 } 2424 2425 *hw_id = params->mqprio.channel.hw_id[tc]; 2426 return 0; 2427} 2428 2429static int mlx5e_open_sqs(struct mlx5e_channel *c, 2430 struct mlx5e_params *params, 2431 struct mlx5e_channel_param *cparam) 2432{ 2433 int err, tc; 2434 2435 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { 2436 int txq_ix = c->ix + tc * params->num_channels; 2437 u32 qos_queue_group_id; 2438 u32 tisn; 2439 2440 tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2441 c->lag_port, tc); 2442 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); 2443 if (err) 2444 goto err_close_sqs; 2445 2446 err = mlx5e_open_txqsq(c, tisn, txq_ix, 2447 params, &cparam->txq_sq, &c->sq[tc], tc, 2448 qos_queue_group_id, 2449 &c->priv->channel_stats[c->ix]->sq[tc]); 2450 if (err) 2451 goto err_close_sqs; 2452 } 2453 2454 return 0; 2455 2456err_close_sqs: 2457 for (tc--; tc >= 0; tc--) 2458 mlx5e_close_txqsq(&c->sq[tc]); 2459 2460 return err; 2461} 2462 2463static void mlx5e_close_sqs(struct mlx5e_channel *c) 2464{ 2465 int tc; 2466 2467 for (tc = 0; tc < c->num_tc; tc++) 2468 mlx5e_close_txqsq(&c->sq[tc]); 2469} 2470 2471static int mlx5e_set_sq_maxrate(struct net_device *dev, 2472 struct mlx5e_txqsq *sq, u32 rate) 2473{ 2474 struct mlx5e_priv *priv = netdev_priv(dev); 2475 struct mlx5_core_dev *mdev = priv->mdev; 2476 struct mlx5e_modify_sq_param msp = {0}; 2477 struct mlx5_rate_limit rl = {0}; 2478 u16 rl_index = 0; 2479 int err; 2480 2481 if (rate == sq->rate_limit) 2482 /* nothing to do */ 2483 return 0; 2484 2485 if (sq->rate_limit) { 2486 rl.rate = sq->rate_limit; 2487 /* remove current rl index to free space to next ones */ 2488 mlx5_rl_remove_rate(mdev, &rl); 2489 } 2490 2491 sq->rate_limit = 0; 2492 2493 if (rate) { 2494 rl.rate = rate; 2495 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 2496 if (err) { 2497 netdev_err(dev, "Failed configuring rate %u: %d\n", 2498 rate, err); 2499 return err; 2500 } 2501 } 2502 2503 msp.curr_state = MLX5_SQC_STATE_RDY; 2504 msp.next_state = MLX5_SQC_STATE_RDY; 2505 msp.rl_index = rl_index; 2506 msp.rl_update = true; 2507 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 2508 if (err) { 2509 netdev_err(dev, "Failed configuring rate %u: %d\n", 2510 rate, err); 2511 /* remove the rate from the table */ 2512 if (rate) 2513 mlx5_rl_remove_rate(mdev, &rl); 2514 return err; 2515 } 2516 2517 sq->rate_limit = rate; 2518 return 0; 2519} 2520 2521static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2522{ 2523 struct mlx5e_priv *priv = netdev_priv(dev); 2524 struct mlx5_core_dev *mdev = priv->mdev; 2525 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 2526 int err = 0; 2527 2528 if (!mlx5_rl_is_supported(mdev)) { 2529 netdev_err(dev, "Rate limiting is not supported on this device\n"); 2530 return -EINVAL; 2531 } 2532 2533 /* rate is given in Mb/sec, HW config is in Kb/sec */ 2534 rate = rate << 10; 2535 2536 /* Check whether rate in valid range, 0 is always valid */ 2537 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 2538 netdev_err(dev, "TX rate %u, is not in range\n", rate); 2539 return -ERANGE; 2540 } 2541 2542 mutex_lock(&priv->state_lock); 2543 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 2544 err = mlx5e_set_sq_maxrate(dev, sq, rate); 2545 if (!err) 2546 priv->tx_rates[index] = rate; 2547 mutex_unlock(&priv->state_lock); 2548 2549 return err; 2550} 2551 2552static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 2553 struct mlx5e_rq_param *rq_params) 2554{ 2555 u16 q_counter = c->priv->q_counter[c->sd_ix]; 2556 int err; 2557 2558 err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); 2559 if (err) 2560 return err; 2561 2562 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); 2563} 2564 2565static int mlx5e_open_queues(struct mlx5e_channel *c, 2566 struct mlx5e_params *params, 2567 struct mlx5e_channel_param *cparam) 2568{ 2569 const struct net_device_ops *netdev_ops = c->netdev->netdev_ops; 2570 struct dim_cq_moder icocq_moder = {0, 0}; 2571 struct mlx5e_create_cq_param ccp; 2572 int err; 2573 2574 mlx5e_build_create_cq_param(&ccp, c); 2575 2576 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, &ccp, 2577 &c->async_icosq.cq); 2578 if (err) 2579 return err; 2580 2581 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, 2582 &c->icosq.cq); 2583 if (err) 2584 goto err_close_async_icosq_cq; 2585 2586 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2587 if (err) 2588 goto err_close_icosq_cq; 2589 2590 if (netdev_ops->ndo_xdp_xmit && c->xdp) { 2591 c->xdpsq = mlx5e_open_xdpredirect_sq(c, params, cparam, &ccp); 2592 if (IS_ERR(c->xdpsq)) { 2593 err = PTR_ERR(c->xdpsq); 2594 goto err_close_tx_cqs; 2595 } 2596 } 2597 2598 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 2599 &c->rq.cq); 2600 if (err) 2601 goto err_close_xdpredirect_sq; 2602 2603 err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, 2604 &ccp, &c->rq_xdpsq.cq) : 0; 2605 if (err) 2606 goto err_close_rx_cq; 2607 2608 spin_lock_init(&c->async_icosq_lock); 2609 2610 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq, 2611 mlx5e_async_icosq_err_cqe_work); 2612 if (err) 2613 goto err_close_rq_xdpsq_cq; 2614 2615 mutex_init(&c->icosq_recovery_lock); 2616 2617 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq, 2618 mlx5e_icosq_err_cqe_work); 2619 if (err) 2620 goto err_close_async_icosq; 2621 2622 err = mlx5e_open_sqs(c, params, cparam); 2623 if (err) 2624 goto err_close_icosq; 2625 2626 err = mlx5e_open_rxq_rq(c, params, &cparam->rq); 2627 if (err) 2628 goto err_close_sqs; 2629 2630 if (c->xdp) { 2631 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 2632 &c->rq_xdpsq, false); 2633 if (err) 2634 goto err_close_rq; 2635 } 2636 2637 return 0; 2638 2639err_close_rq: 2640 mlx5e_close_rq(&c->rq); 2641 2642err_close_sqs: 2643 mlx5e_close_sqs(c); 2644 2645err_close_icosq: 2646 mlx5e_close_icosq(&c->icosq); 2647 2648err_close_async_icosq: 2649 mlx5e_close_icosq(&c->async_icosq); 2650 2651err_close_rq_xdpsq_cq: 2652 if (c->xdp) 2653 mlx5e_close_cq(&c->rq_xdpsq.cq); 2654 2655err_close_rx_cq: 2656 mlx5e_close_cq(&c->rq.cq); 2657 2658err_close_xdpredirect_sq: 2659 if (c->xdpsq) 2660 mlx5e_close_xdpredirect_sq(c->xdpsq); 2661 2662err_close_tx_cqs: 2663 mlx5e_close_tx_cqs(c); 2664 2665err_close_icosq_cq: 2666 mlx5e_close_cq(&c->icosq.cq); 2667 2668err_close_async_icosq_cq: 2669 mlx5e_close_cq(&c->async_icosq.cq); 2670 2671 return err; 2672} 2673 2674static void mlx5e_close_queues(struct mlx5e_channel *c) 2675{ 2676 if (c->xdp) 2677 mlx5e_close_xdpsq(&c->rq_xdpsq); 2678 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */ 2679 cancel_work_sync(&c->icosq.recover_work); 2680 mlx5e_close_rq(&c->rq); 2681 mlx5e_close_sqs(c); 2682 mlx5e_close_icosq(&c->icosq); 2683 mutex_destroy(&c->icosq_recovery_lock); 2684 mlx5e_close_icosq(&c->async_icosq); 2685 if (c->xdp) 2686 mlx5e_close_cq(&c->rq_xdpsq.cq); 2687 mlx5e_close_cq(&c->rq.cq); 2688 if (c->xdpsq) 2689 mlx5e_close_xdpredirect_sq(c->xdpsq); 2690 mlx5e_close_tx_cqs(c); 2691 mlx5e_close_cq(&c->icosq.cq); 2692 mlx5e_close_cq(&c->async_icosq.cq); 2693} 2694 2695static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 2696{ 2697 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 2698 2699 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 2700} 2701 2702static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) 2703{ 2704 if (ix > priv->stats_nch) { 2705 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix, 2706 priv->stats_nch); 2707 return -EINVAL; 2708 } 2709 2710 if (priv->channel_stats[ix]) 2711 return 0; 2712 2713 /* Asymmetric dynamic memory allocation. 2714 * Freed in mlx5e_priv_arrays_free, not on channel closure. 2715 */ 2716 netdev_dbg(priv->netdev, "Creating channel stats %d\n", ix); 2717 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats), 2718 GFP_KERNEL, cpu_to_node(cpu)); 2719 if (!priv->channel_stats[ix]) 2720 return -ENOMEM; 2721 priv->stats_nch++; 2722 2723 return 0; 2724} 2725 2726void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2727{ 2728 spin_lock_bh(&c->async_icosq_lock); 2729 mlx5e_trigger_irq(&c->async_icosq); 2730 spin_unlock_bh(&c->async_icosq_lock); 2731} 2732 2733void mlx5e_trigger_napi_sched(struct napi_struct *napi) 2734{ 2735 local_bh_disable(); 2736 napi_schedule(napi); 2737 local_bh_enable(); 2738} 2739 2740static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) 2741{ 2742 struct mlx5e_hw_objs *hw_objs = &c->mdev->mlx5e_res.hw_objs; 2743 2744 /* No dedicated Ethernet doorbells, use the global one. */ 2745 if (hw_objs->num_bfregs == 0) { 2746 c->bfreg = &c->mdev->priv.bfreg; 2747 return; 2748 } 2749 2750 /* Round-robin between doorbells. */ 2751 c->bfreg = hw_objs->bfregs + c->vec_ix % hw_objs->num_bfregs; 2752} 2753 2754static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 2755 struct mlx5e_params *params, 2756 struct xsk_buff_pool *xsk_pool, 2757 struct mlx5e_channel **cp) 2758{ 2759 struct net_device *netdev = priv->netdev; 2760 struct mlx5e_channel_param *cparam; 2761 struct mlx5_core_dev *mdev; 2762 struct mlx5e_xsk_param xsk; 2763 struct mlx5e_channel *c; 2764 unsigned int irq; 2765 int vec_ix; 2766 int cpu; 2767 int err; 2768 2769 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 2770 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 2771 cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); 2772 2773 err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); 2774 if (err) 2775 return err; 2776 2777 err = mlx5e_channel_stats_alloc(priv, ix, cpu); 2778 if (err) 2779 return err; 2780 2781 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 2782 cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL); 2783 if (!c || !cparam) { 2784 err = -ENOMEM; 2785 goto err_free; 2786 } 2787 2788 err = mlx5e_build_channel_param(mdev, params, cparam); 2789 if (err) 2790 goto err_free; 2791 2792 c->priv = priv; 2793 c->mdev = mdev; 2794 c->ix = ix; 2795 c->vec_ix = vec_ix; 2796 c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); 2797 c->cpu = cpu; 2798 c->pdev = mlx5_core_dma_dev(mdev); 2799 c->netdev = priv->netdev; 2800 c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 2801 c->num_tc = mlx5e_get_dcb_num_tc(params); 2802 c->xdp = !!params->xdp_prog; 2803 c->stats = &priv->channel_stats[ix]->ch; 2804 c->aff_mask = irq_get_effective_affinity_mask(irq); 2805 c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); 2806 2807 mlx5e_channel_pick_doorbell(c); 2808 2809 netif_napi_add_config_locked(netdev, &c->napi, mlx5e_napi_poll, ix); 2810 netif_napi_set_irq_locked(&c->napi, irq); 2811 2812 err = mlx5e_open_queues(c, params, cparam); 2813 if (unlikely(err)) 2814 goto err_napi_del; 2815 2816 if (xsk_pool) { 2817 mlx5e_build_xsk_param(xsk_pool, &xsk); 2818 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c); 2819 if (unlikely(err)) 2820 goto err_close_queues; 2821 } 2822 2823 *cp = c; 2824 2825 kvfree(cparam); 2826 return 0; 2827 2828err_close_queues: 2829 mlx5e_close_queues(c); 2830 2831err_napi_del: 2832 netif_napi_del_locked(&c->napi); 2833 2834err_free: 2835 kvfree(cparam); 2836 kvfree(c); 2837 2838 return err; 2839} 2840 2841static void mlx5e_activate_channel(struct mlx5e_channel *c) 2842{ 2843 int tc; 2844 2845 napi_enable_locked(&c->napi); 2846 2847 for (tc = 0; tc < c->num_tc; tc++) 2848 mlx5e_activate_txqsq(&c->sq[tc]); 2849 mlx5e_activate_icosq(&c->icosq); 2850 mlx5e_activate_icosq(&c->async_icosq); 2851 2852 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2853 mlx5e_activate_xsk(c); 2854 else 2855 mlx5e_activate_rq(&c->rq); 2856 2857 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi); 2858} 2859 2860static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2861{ 2862 int tc; 2863 2864 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL); 2865 2866 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2867 mlx5e_deactivate_xsk(c); 2868 else 2869 mlx5e_deactivate_rq(&c->rq); 2870 2871 mlx5e_deactivate_icosq(&c->async_icosq); 2872 mlx5e_deactivate_icosq(&c->icosq); 2873 for (tc = 0; tc < c->num_tc; tc++) 2874 mlx5e_deactivate_txqsq(&c->sq[tc]); 2875 mlx5e_qos_deactivate_queues(c); 2876 2877 napi_disable_locked(&c->napi); 2878} 2879 2880static void mlx5e_close_channel(struct mlx5e_channel *c) 2881{ 2882 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2883 mlx5e_close_xsk(c); 2884 mlx5e_close_queues(c); 2885 mlx5e_qos_close_queues(c); 2886 netif_napi_del_locked(&c->napi); 2887 2888 kvfree(c); 2889} 2890 2891int mlx5e_open_channels(struct mlx5e_priv *priv, 2892 struct mlx5e_channels *chs) 2893{ 2894 int err = -ENOMEM; 2895 int i; 2896 2897 chs->num = chs->params.num_channels; 2898 2899 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2900 if (!chs->c) 2901 goto err_out; 2902 2903 for (i = 0; i < chs->num; i++) { 2904 struct xsk_buff_pool *xsk_pool = NULL; 2905 2906 if (chs->params.xdp_prog) 2907 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); 2908 2909 err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]); 2910 if (err) 2911 goto err_close_channels; 2912 } 2913 2914 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) { 2915 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); 2916 if (err) 2917 goto err_close_channels; 2918 } 2919 2920 if (priv->htb) { 2921 err = mlx5e_qos_open_queues(priv, chs); 2922 if (err) 2923 goto err_close_ptp; 2924 } 2925 2926 mlx5e_health_channels_update(priv); 2927 return 0; 2928 2929err_close_ptp: 2930 if (chs->ptp) 2931 mlx5e_ptp_close(chs->ptp); 2932 2933err_close_channels: 2934 for (i--; i >= 0; i--) 2935 mlx5e_close_channel(chs->c[i]); 2936 2937 kfree(chs->c); 2938err_out: 2939 chs->num = 0; 2940 return err; 2941} 2942 2943static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 2944{ 2945 int i; 2946 2947 for (i = 0; i < chs->num; i++) 2948 mlx5e_activate_channel(chs->c[i]); 2949 2950 if (priv->htb) 2951 mlx5e_qos_activate_queues(priv); 2952 2953 for (i = 0; i < chs->num; i++) 2954 mlx5e_trigger_napi_icosq(chs->c[i]); 2955 2956 if (chs->ptp) 2957 mlx5e_ptp_activate_channel(chs->ptp); 2958} 2959 2960static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2961{ 2962 int err = 0; 2963 int i; 2964 2965 for (i = 0; i < chs->num; i++) { 2966 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2967 struct mlx5e_channel *c = chs->c[i]; 2968 2969 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2970 continue; 2971 2972 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); 2973 2974 /* Don't wait on the XSK RQ, because the newer xdpsock sample 2975 * doesn't provide any Fill Ring entries at the setup stage. 2976 */ 2977 } 2978 2979 return err ? -ETIMEDOUT : 0; 2980} 2981 2982static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2983{ 2984 int i; 2985 2986 if (chs->ptp) 2987 mlx5e_ptp_deactivate_channel(chs->ptp); 2988 2989 for (i = 0; i < chs->num; i++) 2990 mlx5e_deactivate_channel(chs->c[i]); 2991} 2992 2993void mlx5e_close_channels(struct mlx5e_channels *chs) 2994{ 2995 int i; 2996 2997 ASSERT_RTNL(); 2998 if (chs->ptp) { 2999 mlx5e_ptp_close(chs->ptp); 3000 chs->ptp = NULL; 3001 } 3002 for (i = 0; i < chs->num; i++) 3003 mlx5e_close_channel(chs->c[i]); 3004 3005 kfree(chs->c); 3006 chs->num = 0; 3007} 3008 3009static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv) 3010{ 3011 struct mlx5e_rx_res *res = priv->rx_res; 3012 3013 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge); 3014} 3015 3016static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge); 3017 3018static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 3019 struct mlx5e_params *params, u16 mtu) 3020{ 3021 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 3022 int err; 3023 3024 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 3025 if (err) 3026 return err; 3027 3028 /* Update vport context MTU */ 3029 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 3030 return 0; 3031} 3032 3033static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 3034 struct mlx5e_params *params, u16 *mtu) 3035{ 3036 u16 hw_mtu = 0; 3037 int err; 3038 3039 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 3040 if (err || !hw_mtu) /* fallback to port oper mtu */ 3041 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 3042 3043 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 3044} 3045 3046int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 3047{ 3048 struct mlx5e_params *params = &priv->channels.params; 3049 struct net_device *netdev = priv->netdev; 3050 struct mlx5_core_dev *mdev = priv->mdev; 3051 u16 mtu; 3052 int err; 3053 3054 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 3055 if (err) 3056 return err; 3057 3058 mlx5e_query_mtu(mdev, params, &mtu); 3059 if (mtu != params->sw_mtu) 3060 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 3061 __func__, mtu, params->sw_mtu); 3062 3063 params->sw_mtu = mtu; 3064 return 0; 3065} 3066 3067MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 3068 3069void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 3070{ 3071 struct mlx5e_params *params = &priv->channels.params; 3072 struct net_device *netdev = priv->netdev; 3073 struct mlx5_core_dev *mdev = priv->mdev; 3074 u16 max_mtu; 3075 3076 /* MTU range: 68 - hw-specific max */ 3077 netdev->min_mtu = ETH_MIN_MTU; 3078 3079 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 3080 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 3081 ETH_MAX_MTU); 3082} 3083 3084static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, 3085 struct netdev_tc_txq *tc_to_txq) 3086{ 3087 int tc, err; 3088 3089 netdev_reset_tc(netdev); 3090 3091 if (ntc == 1) 3092 return 0; 3093 3094 err = netdev_set_num_tc(netdev, ntc); 3095 if (err) { 3096 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc); 3097 return err; 3098 } 3099 3100 for (tc = 0; tc < ntc; tc++) { 3101 u16 count, offset; 3102 3103 count = tc_to_txq[tc].count; 3104 offset = tc_to_txq[tc].offset; 3105 netdev_set_tc_queue(netdev, tc, count, offset); 3106 } 3107 3108 return 0; 3109} 3110 3111int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) 3112{ 3113 int nch, ntc, num_txqs, err; 3114 int qos_queues = 0; 3115 3116 if (priv->htb) 3117 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); 3118 3119 nch = priv->channels.params.num_channels; 3120 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3121 num_txqs = nch * ntc + qos_queues; 3122 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) 3123 num_txqs += ntc; 3124 3125 netdev_dbg(priv->netdev, "Setting num_txqs %d\n", num_txqs); 3126 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); 3127 if (err) 3128 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 3129 3130 return err; 3131} 3132 3133static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 3134 struct mlx5e_params *params) 3135{ 3136 int ix; 3137 3138 for (ix = 0; ix < params->num_channels; ix++) { 3139 int num_comp_vectors, irq, vec_ix; 3140 struct mlx5_core_dev *mdev; 3141 3142 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 3143 num_comp_vectors = mlx5_comp_vectors_max(mdev); 3144 cpumask_clear(priv->scratchpad.cpumask); 3145 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 3146 3147 for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { 3148 int cpu = mlx5_comp_vector_get_cpu(mdev, irq); 3149 3150 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 3151 } 3152 3153 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 3154 } 3155} 3156 3157static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv) 3158{ 3159 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; 3160 struct net_device *netdev = priv->netdev; 3161 int old_num_txqs, old_ntc; 3162 int nch, ntc; 3163 int err; 3164 int i; 3165 3166 old_num_txqs = netdev->real_num_tx_queues; 3167 old_ntc = netdev->num_tc ? : 1; 3168 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) 3169 old_tc_to_txq[i] = netdev->tc_to_txq[i]; 3170 3171 nch = priv->channels.params.num_channels; 3172 ntc = priv->channels.params.mqprio.num_tc; 3173 tc_to_txq = priv->channels.params.mqprio.tc_to_txq; 3174 3175 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); 3176 if (err) 3177 goto err_out; 3178 err = mlx5e_update_tx_netdev_queues(priv); 3179 if (err) 3180 goto err_tcs; 3181 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 3182 3183 return 0; 3184 3185err_tcs: 3186 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc, 3187 old_tc_to_txq)); 3188err_out: 3189 return err; 3190} 3191 3192MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_tc_and_tx_queues); 3193 3194static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 3195{ 3196 u16 count = priv->channels.params.num_channels; 3197 struct net_device *netdev = priv->netdev; 3198 int old_num_rxqs; 3199 int err; 3200 3201 old_num_rxqs = netdev->real_num_rx_queues; 3202 err = netif_set_real_num_rx_queues(netdev, count); 3203 if (err) { 3204 netdev_warn(netdev, "%s: netif_set_real_num_rx_queues failed, %d\n", 3205 __func__, err); 3206 return err; 3207 } 3208 err = mlx5e_update_tc_and_tx_queues(priv); 3209 if (err) { 3210 /* mlx5e_update_tc_and_tx_queues can fail if channels or TCs number increases. 3211 * Since channel number changed, it increased. That means, the call to 3212 * netif_set_real_num_rx_queues below should not fail, because it 3213 * decreases the number of RX queues. 3214 */ 3215 WARN_ON_ONCE(netif_set_real_num_rx_queues(netdev, old_num_rxqs)); 3216 return err; 3217 } 3218 3219 /* This function may be called on attach, before priv->rx_res is created. */ 3220 if (priv->rx_res) { 3221 mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count); 3222 3223 if (!netif_is_rxfh_configured(priv->netdev)) 3224 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count); 3225 } 3226 3227 return 0; 3228} 3229 3230MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 3231 3232static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 3233{ 3234 int i, ch, tc, num_tc; 3235 3236 ch = priv->channels.num; 3237 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3238 3239 for (i = 0; i < ch; i++) { 3240 for (tc = 0; tc < num_tc; tc++) { 3241 struct mlx5e_channel *c = priv->channels.c[i]; 3242 struct mlx5e_txqsq *sq = &c->sq[tc]; 3243 3244 priv->txq2sq[sq->txq_ix] = sq; 3245 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3246 } 3247 } 3248 3249 if (!priv->channels.ptp) 3250 goto out; 3251 3252 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state)) 3253 goto out; 3254 3255 for (tc = 0; tc < num_tc; tc++) { 3256 struct mlx5e_ptp *c = priv->channels.ptp; 3257 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq; 3258 3259 priv->txq2sq[sq->txq_ix] = sq; 3260 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3261 } 3262 3263out: 3264 /* Make the change to txq2sq visible before the queue is started. 3265 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 3266 * which pairs with this barrier. 3267 */ 3268 smp_wmb(); 3269} 3270 3271void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 3272{ 3273 mlx5e_build_txq_maps(priv); 3274 mlx5e_activate_channels(priv, &priv->channels); 3275 mlx5e_xdp_tx_enable(priv); 3276 3277 /* dev_watchdog() wants all TX queues to be started when the carrier is 3278 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1. 3279 * Make it happy to avoid TX timeout false alarms. 3280 */ 3281 netif_tx_start_all_queues(priv->netdev); 3282 3283 if (mlx5e_is_vport_rep(priv)) 3284 mlx5e_rep_activate_channels(priv); 3285 3286 set_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3287 3288 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 3289 3290 if (priv->rx_res) 3291 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels); 3292} 3293 3294static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv) 3295{ 3296 WARN_ON_ONCE(test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)); 3297 if (current_work() != &priv->tx_timeout_work) 3298 cancel_work_sync(&priv->tx_timeout_work); 3299} 3300 3301void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 3302{ 3303 if (priv->rx_res) 3304 mlx5e_rx_res_channels_deactivate(priv->rx_res); 3305 3306 clear_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3307 mlx5e_cancel_tx_timeout_work(priv); 3308 3309 if (mlx5e_is_vport_rep(priv)) 3310 mlx5e_rep_deactivate_channels(priv); 3311 3312 /* The results of ndo_select_queue are unreliable, while netdev config 3313 * is being changed (real_num_tx_queues, num_tc). Stop all queues to 3314 * prevent ndo_start_xmit from being called, so that it can assume that 3315 * the selected queue is always valid. 3316 */ 3317 netif_tx_disable(priv->netdev); 3318 3319 mlx5e_xdp_tx_disable(priv); 3320 mlx5e_deactivate_channels(&priv->channels); 3321} 3322 3323static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, 3324 struct mlx5e_params *new_params, 3325 mlx5e_fp_preactivate preactivate, 3326 void *context) 3327{ 3328 struct mlx5e_params old_params; 3329 3330 old_params = priv->channels.params; 3331 priv->channels.params = *new_params; 3332 3333 if (preactivate) { 3334 int err; 3335 3336 err = preactivate(priv, context); 3337 if (err) { 3338 priv->channels.params = old_params; 3339 return err; 3340 } 3341 } 3342 3343 mlx5e_set_xdp_feature(priv); 3344 return 0; 3345} 3346 3347static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 3348 struct mlx5e_channels *old_chs, 3349 struct mlx5e_channels *new_chs, 3350 mlx5e_fp_preactivate preactivate, 3351 void *context) 3352{ 3353 struct net_device *netdev = priv->netdev; 3354 int carrier_ok; 3355 int err = 0; 3356 3357 carrier_ok = netif_carrier_ok(netdev); 3358 netif_carrier_off(netdev); 3359 3360 mlx5e_deactivate_priv_channels(priv); 3361 3362 priv->channels = *new_chs; 3363 3364 /* New channels are ready to roll, call the preactivate hook if needed 3365 * to modify HW settings or update kernel parameters. 3366 */ 3367 if (preactivate) { 3368 err = preactivate(priv, context); 3369 if (err) { 3370 priv->channels = *old_chs; 3371 goto out; 3372 } 3373 } 3374 3375 mlx5e_set_xdp_feature(priv); 3376 if (!MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3377 mlx5e_close_channels(old_chs); 3378 priv->profile->update_rx(priv); 3379 3380 mlx5e_selq_apply(&priv->selq); 3381out: 3382 mlx5e_activate_priv_channels(priv); 3383 3384 /* return carrier back if needed */ 3385 if (carrier_ok) 3386 netif_carrier_on(netdev); 3387 3388 return err; 3389} 3390 3391int mlx5e_safe_switch_params(struct mlx5e_priv *priv, 3392 struct mlx5e_params *params, 3393 mlx5e_fp_preactivate preactivate, 3394 void *context, bool reset) 3395{ 3396 struct mlx5e_channels *old_chs, *new_chs; 3397 int err; 3398 3399 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); 3400 if (!reset) 3401 return mlx5e_switch_priv_params(priv, params, preactivate, context); 3402 3403 old_chs = kzalloc(sizeof(*old_chs), GFP_KERNEL); 3404 new_chs = kzalloc(sizeof(*new_chs), GFP_KERNEL); 3405 if (!old_chs || !new_chs) { 3406 err = -ENOMEM; 3407 goto err_free_chs; 3408 } 3409 3410 new_chs->params = *params; 3411 3412 mlx5e_selq_prepare_params(&priv->selq, &new_chs->params); 3413 3414 err = mlx5e_open_channels(priv, new_chs); 3415 if (err) 3416 goto err_cancel_selq; 3417 3418 *old_chs = priv->channels; 3419 3420 err = mlx5e_switch_priv_channels(priv, old_chs, new_chs, 3421 preactivate, context); 3422 if (err) 3423 goto err_close; 3424 3425 if (MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3426 mlx5e_close_channels(old_chs); 3427 3428 kfree(new_chs); 3429 kfree(old_chs); 3430 return 0; 3431 3432err_close: 3433 mlx5e_close_channels(new_chs); 3434 3435err_cancel_selq: 3436 mlx5e_selq_cancel(&priv->selq); 3437err_free_chs: 3438 kfree(new_chs); 3439 kfree(old_chs); 3440 return err; 3441} 3442 3443int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3444{ 3445 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true); 3446} 3447 3448void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3449{ 3450 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3451 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3452} 3453 3454static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3455 enum mlx5_port_status state) 3456{ 3457 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3458 int vport_admin_state; 3459 3460 mlx5_set_port_admin_status(mdev, state); 3461 3462 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS || 3463 !MLX5_CAP_GEN(mdev, uplink_follow)) 3464 return; 3465 3466 if (state == MLX5_PORT_UP) 3467 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3468 else 3469 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3470 3471 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3472} 3473 3474int mlx5e_open_locked(struct net_device *netdev) 3475{ 3476 struct mlx5e_priv *priv = netdev_priv(netdev); 3477 int err; 3478 3479 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); 3480 3481 set_bit(MLX5E_STATE_OPENED, &priv->state); 3482 3483 err = mlx5e_open_channels(priv, &priv->channels); 3484 if (err) 3485 goto err_clear_state_opened_flag; 3486 3487 err = priv->profile->update_rx(priv); 3488 if (err) 3489 goto err_close_channels; 3490 3491 mlx5e_selq_apply(&priv->selq); 3492 mlx5e_activate_priv_channels(priv); 3493 mlx5e_apply_traps(priv, true); 3494 if (priv->profile->update_carrier) 3495 priv->profile->update_carrier(priv); 3496 3497 mlx5e_queue_update_stats(priv); 3498 return 0; 3499 3500err_close_channels: 3501 mlx5e_close_channels(&priv->channels); 3502err_clear_state_opened_flag: 3503 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3504 mlx5e_selq_cancel(&priv->selq); 3505 return err; 3506} 3507 3508int mlx5e_open(struct net_device *netdev) 3509{ 3510 struct mlx5e_priv *priv = netdev_priv(netdev); 3511 int err; 3512 3513 mutex_lock(&priv->state_lock); 3514 err = mlx5e_open_locked(netdev); 3515 if (!err) 3516 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3517 mutex_unlock(&priv->state_lock); 3518 3519 return err; 3520} 3521 3522int mlx5e_close_locked(struct net_device *netdev) 3523{ 3524 struct mlx5e_priv *priv = netdev_priv(netdev); 3525 3526 /* May already be CLOSED in case a previous configuration operation 3527 * (e.g RX/TX queue size change) that involves close&open failed. 3528 */ 3529 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3530 return 0; 3531 3532 mlx5e_apply_traps(priv, false); 3533 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3534 3535 netif_carrier_off(priv->netdev); 3536 mlx5e_deactivate_priv_channels(priv); 3537 mlx5e_close_channels(&priv->channels); 3538 3539 return 0; 3540} 3541 3542int mlx5e_close(struct net_device *netdev) 3543{ 3544 struct mlx5e_priv *priv = netdev_priv(netdev); 3545 int err; 3546 3547 if (!netif_device_present(netdev)) 3548 return -ENODEV; 3549 3550 mutex_lock(&priv->state_lock); 3551 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3552 err = mlx5e_close_locked(netdev); 3553 mutex_unlock(&priv->state_lock); 3554 3555 return err; 3556} 3557 3558static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) 3559{ 3560 mlx5_wq_destroy(&rq->wq_ctrl); 3561} 3562 3563static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3564 struct mlx5e_rq *rq, 3565 struct mlx5e_rq_param *param) 3566{ 3567 void *rqc = param->rqc; 3568 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 3569 int err; 3570 3571 param->wq.db_numa_node = param->wq.buf_numa_node; 3572 3573 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 3574 &rq->wq_ctrl); 3575 if (err) 3576 return err; 3577 3578 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3579 xdp_rxq_info_unused(&rq->xdp_rxq); 3580 3581 rq->mdev = mdev; 3582 3583 return 0; 3584} 3585 3586static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, 3587 struct mlx5e_cq *cq, 3588 struct mlx5e_cq_param *param) 3589{ 3590 struct mlx5_core_dev *mdev = priv->mdev; 3591 3592 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3593 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3594 3595 return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, 3596 mdev->priv.bfreg.up, param, cq); 3597} 3598 3599int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3600 struct mlx5e_rq *drop_rq) 3601{ 3602 struct mlx5_core_dev *mdev = priv->mdev; 3603 struct mlx5e_cq_param cq_param = {}; 3604 struct mlx5e_rq_param rq_param = {}; 3605 struct mlx5e_cq *cq = &drop_rq->cq; 3606 int err; 3607 3608 mlx5e_build_drop_rq_param(mdev, &rq_param); 3609 3610 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); 3611 if (err) 3612 return err; 3613 3614 err = mlx5e_create_cq(cq, &cq_param); 3615 if (err) 3616 goto err_free_cq; 3617 3618 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3619 if (err) 3620 goto err_destroy_cq; 3621 3622 err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); 3623 if (err) 3624 goto err_free_rq; 3625 3626 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3627 if (err) 3628 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3629 3630 return 0; 3631 3632err_free_rq: 3633 mlx5e_free_drop_rq(drop_rq); 3634 3635err_destroy_cq: 3636 mlx5e_destroy_cq(cq); 3637 3638err_free_cq: 3639 mlx5e_free_cq(cq); 3640 3641 return err; 3642} 3643 3644void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3645{ 3646 mlx5e_destroy_rq(drop_rq); 3647 mlx5e_free_drop_rq(drop_rq); 3648 mlx5e_destroy_cq(&drop_rq->cq); 3649 mlx5e_free_cq(&drop_rq->cq); 3650} 3651 3652static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3653{ 3654 if (priv->mqprio_rl) { 3655 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3656 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3657 priv->mqprio_rl = NULL; 3658 } 3659 mlx5e_accel_cleanup_tx(priv); 3660} 3661 3662static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3663{ 3664 int err; 3665 int i; 3666 3667 for (i = 0; i < chs->num; i++) { 3668 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3669 if (err) 3670 return err; 3671 } 3672 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state)) 3673 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd); 3674 3675 return 0; 3676} 3677 3678static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3679 int ntc, int nch) 3680{ 3681 int tc; 3682 3683 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE); 3684 3685 /* Map netdev TCs to offset 0. 3686 * We have our own UP to TXQ mapping for DCB mode of QoS 3687 */ 3688 for (tc = 0; tc < ntc; tc++) { 3689 tc_to_txq[tc] = (struct netdev_tc_txq) { 3690 .count = nch, 3691 .offset = 0, 3692 }; 3693 } 3694} 3695 3696static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3697 struct tc_mqprio_qopt *qopt) 3698{ 3699 int tc; 3700 3701 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3702 tc_to_txq[tc] = (struct netdev_tc_txq) { 3703 .count = qopt->count[tc], 3704 .offset = qopt->offset[tc], 3705 }; 3706 } 3707} 3708 3709static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) 3710{ 3711 params->mqprio.mode = TC_MQPRIO_MODE_DCB; 3712 params->mqprio.num_tc = num_tc; 3713 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, 3714 params->num_channels); 3715} 3716 3717static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, 3718 struct mlx5e_mqprio_rl *rl) 3719{ 3720 int tc; 3721 3722 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3723 u32 hw_id = 0; 3724 3725 if (rl) 3726 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); 3727 params->mqprio.channel.hw_id[tc] = hw_id; 3728 } 3729} 3730 3731static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, 3732 struct tc_mqprio_qopt_offload *mqprio, 3733 struct mlx5e_mqprio_rl *rl) 3734{ 3735 int tc; 3736 3737 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; 3738 params->mqprio.num_tc = mqprio->qopt.num_tc; 3739 3740 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 3741 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; 3742 3743 mlx5e_mqprio_rl_update_params(params, rl); 3744 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); 3745} 3746 3747static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) 3748{ 3749 mlx5e_params_mqprio_dcb_set(params, 1); 3750} 3751 3752static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, 3753 struct tc_mqprio_qopt *mqprio) 3754{ 3755 struct mlx5e_params new_params; 3756 u8 tc = mqprio->num_tc; 3757 int err; 3758 3759 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3760 3761 if (tc && tc != MLX5_MAX_NUM_TC) 3762 return -EINVAL; 3763 3764 new_params = priv->channels.params; 3765 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1); 3766 3767 err = mlx5e_safe_switch_params(priv, &new_params, 3768 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3769 3770 if (!err && priv->mqprio_rl) { 3771 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3772 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3773 priv->mqprio_rl = NULL; 3774 } 3775 3776 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3777 mlx5e_get_dcb_num_tc(&priv->channels.params)); 3778 return err; 3779} 3780 3781static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, 3782 struct tc_mqprio_qopt_offload *mqprio) 3783{ 3784 struct net_device *netdev = priv->netdev; 3785 struct mlx5e_ptp *ptp_channel; 3786 int agg_count = 0; 3787 int i; 3788 3789 ptp_channel = priv->channels.ptp; 3790 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) { 3791 netdev_err(netdev, 3792 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n"); 3793 return -EINVAL; 3794 } 3795 3796 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 || 3797 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC) 3798 return -EINVAL; 3799 3800 for (i = 0; i < mqprio->qopt.num_tc; i++) { 3801 if (!mqprio->qopt.count[i]) { 3802 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i); 3803 return -EINVAL; 3804 } 3805 if (mqprio->min_rate[i]) { 3806 netdev_err(netdev, "Min tx rate is not supported\n"); 3807 return -EINVAL; 3808 } 3809 3810 if (mqprio->max_rate[i]) { 3811 int err; 3812 3813 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]); 3814 if (err) 3815 return err; 3816 } 3817 3818 if (mqprio->qopt.offset[i] != agg_count) { 3819 netdev_err(netdev, "Discontinuous queues config is not supported\n"); 3820 return -EINVAL; 3821 } 3822 agg_count += mqprio->qopt.count[i]; 3823 } 3824 3825 if (priv->channels.params.num_channels != agg_count) { 3826 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n", 3827 agg_count, priv->channels.params.num_channels); 3828 return -EINVAL; 3829 } 3830 3831 return 0; 3832} 3833 3834static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) 3835{ 3836 int tc; 3837 3838 for (tc = 0; tc < num_tc; tc++) 3839 if (max_rate[tc]) 3840 return true; 3841 return false; 3842} 3843 3844static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, 3845 u8 num_tc, u64 max_rate[]) 3846{ 3847 struct mlx5e_mqprio_rl *rl; 3848 int err; 3849 3850 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) 3851 return NULL; 3852 3853 rl = mlx5e_mqprio_rl_alloc(); 3854 if (!rl) 3855 return ERR_PTR(-ENOMEM); 3856 3857 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); 3858 if (err) { 3859 mlx5e_mqprio_rl_free(rl); 3860 return ERR_PTR(err); 3861 } 3862 3863 return rl; 3864} 3865 3866static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, 3867 struct tc_mqprio_qopt_offload *mqprio) 3868{ 3869 struct mlx5e_params new_params; 3870 struct mlx5e_mqprio_rl *rl; 3871 int err; 3872 3873 err = mlx5e_mqprio_channel_validate(priv, mqprio); 3874 if (err) 3875 return err; 3876 3877 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); 3878 if (IS_ERR(rl)) 3879 return PTR_ERR(rl); 3880 3881 new_params = priv->channels.params; 3882 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); 3883 3884 err = mlx5e_safe_switch_params(priv, &new_params, 3885 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3886 if (err) { 3887 if (rl) { 3888 mlx5e_mqprio_rl_cleanup(rl); 3889 mlx5e_mqprio_rl_free(rl); 3890 } 3891 return err; 3892 } 3893 3894 if (priv->mqprio_rl) { 3895 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3896 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3897 } 3898 priv->mqprio_rl = rl; 3899 3900 return 0; 3901} 3902 3903static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3904 struct tc_mqprio_qopt_offload *mqprio) 3905{ 3906 /* MQPRIO is another toplevel qdisc that can't be attached 3907 * simultaneously with the offloaded HTB. 3908 */ 3909 if (mlx5e_selq_is_htb_enabled(&priv->selq)) { 3910 NL_SET_ERR_MSG_MOD(mqprio->extack, 3911 "MQPRIO cannot be configured when HTB offload is enabled."); 3912 return -EOPNOTSUPP; 3913 } 3914 3915 switch (mqprio->mode) { 3916 case TC_MQPRIO_MODE_DCB: 3917 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt); 3918 case TC_MQPRIO_MODE_CHANNEL: 3919 return mlx5e_setup_tc_mqprio_channel(priv, mqprio); 3920 default: 3921 return -EOPNOTSUPP; 3922 } 3923} 3924 3925static LIST_HEAD(mlx5e_block_cb_list); 3926 3927static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3928 void *type_data) 3929{ 3930 struct mlx5e_priv *priv = netdev_priv(dev); 3931 bool tc_unbind = false; 3932 int err; 3933 3934 if (type == TC_SETUP_BLOCK && 3935 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND) 3936 tc_unbind = true; 3937 3938 if (!netif_device_present(dev) && !tc_unbind) 3939 return -ENODEV; 3940 3941 switch (type) { 3942 case TC_SETUP_BLOCK: { 3943 struct flow_block_offload *f = type_data; 3944 3945 f->unlocked_driver_cb = true; 3946 return flow_block_cb_setup_simple(type_data, 3947 &mlx5e_block_cb_list, 3948 mlx5e_setup_tc_block_cb, 3949 priv, priv, true); 3950 } 3951 case TC_SETUP_QDISC_MQPRIO: 3952 mutex_lock(&priv->state_lock); 3953 err = mlx5e_setup_tc_mqprio(priv, type_data); 3954 mutex_unlock(&priv->state_lock); 3955 return err; 3956 case TC_SETUP_QDISC_HTB: 3957 mutex_lock(&priv->state_lock); 3958 err = mlx5e_htb_setup_tc(priv, type_data); 3959 mutex_unlock(&priv->state_lock); 3960 return err; 3961 default: 3962 return -EOPNOTSUPP; 3963 } 3964} 3965 3966void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3967{ 3968 int i; 3969 3970 for (i = 0; i < priv->stats_nch; i++) { 3971 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 3972 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 3973 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3974 int j; 3975 3976 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 3977 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 3978 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets; 3979 3980 for (j = 0; j < priv->max_opened_tc; j++) { 3981 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3982 3983 s->tx_packets += sq_stats->packets; 3984 s->tx_bytes += sq_stats->bytes; 3985 s->tx_dropped += sq_stats->dropped; 3986 } 3987 } 3988 if (priv->tx_ptp_opened) { 3989 for (i = 0; i < priv->max_opened_tc; i++) { 3990 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i]; 3991 3992 s->tx_packets += sq_stats->packets; 3993 s->tx_bytes += sq_stats->bytes; 3994 s->tx_dropped += sq_stats->dropped; 3995 } 3996 } 3997 if (priv->rx_ptp_opened) { 3998 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 3999 4000 s->rx_packets += rq_stats->packets; 4001 s->rx_bytes += rq_stats->bytes; 4002 s->multicast += rq_stats->mcast_packets; 4003 } 4004 4005#ifdef CONFIG_MLX5_EN_PSP 4006 if (priv->psp) 4007 s->tx_dropped += atomic_read(&priv->psp->tx_drop); 4008#endif 4009} 4010 4011void 4012mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 4013{ 4014 struct mlx5e_priv *priv = netdev_priv(dev); 4015 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 4016 4017 if (!netif_device_present(dev)) 4018 return; 4019 4020 /* In switchdev mode, monitor counters doesn't monitor 4021 * rx/tx stats of 802_3. The update stats mechanism 4022 * should keep the 802_3 layout counters updated 4023 */ 4024 if (!mlx5e_monitor_counter_supported(priv) || 4025 mlx5e_is_uplink_rep(priv)) { 4026 /* update HW stats in background for next time */ 4027 mlx5e_queue_update_stats(priv); 4028 } 4029 4030 netdev_stats_to_stats64(stats, &dev->stats); 4031 4032 if (mlx5e_is_uplink_rep(priv)) { 4033 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 4034 4035 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 4036 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 4037 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 4038 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 4039 4040 /* vport multicast also counts packets that are dropped due to steering 4041 * or rx out of buffer 4042 */ 4043 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 4044 } else { 4045 mlx5e_fold_sw_stats64(priv, stats); 4046 } 4047 4048 stats->rx_missed_errors += priv->stats.qcnt.rx_out_of_buffer; 4049 stats->rx_dropped += PPORT_2863_GET(pstats, if_in_discards); 4050 4051 stats->rx_length_errors += 4052 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 4053 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 4054 PPORT_802_3_GET(pstats, a_frame_too_long_errors) + 4055 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); 4056 stats->rx_crc_errors += 4057 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 4058 stats->rx_frame_errors += PPORT_802_3_GET(pstats, a_alignment_errors); 4059 stats->tx_aborted_errors += PPORT_2863_GET(pstats, if_out_discards); 4060 stats->rx_errors += stats->rx_length_errors + stats->rx_crc_errors + 4061 stats->rx_frame_errors; 4062 stats->tx_errors += stats->tx_aborted_errors + stats->tx_carrier_errors; 4063} 4064 4065static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) 4066{ 4067 if (mlx5e_is_uplink_rep(priv)) 4068 return; /* no rx mode for uplink rep */ 4069 4070 queue_work(priv->wq, &priv->set_rx_mode_work); 4071} 4072 4073static void mlx5e_set_rx_mode(struct net_device *dev) 4074{ 4075 struct mlx5e_priv *priv = netdev_priv(dev); 4076 4077 mlx5e_nic_set_rx_mode(priv); 4078} 4079 4080static int mlx5e_set_mac(struct net_device *netdev, void *addr) 4081{ 4082 struct mlx5e_priv *priv = netdev_priv(netdev); 4083 struct sockaddr *saddr = addr; 4084 4085 if (!is_valid_ether_addr(saddr->sa_data)) 4086 return -EADDRNOTAVAIL; 4087 4088 netif_addr_lock_bh(netdev); 4089 eth_hw_addr_set(netdev, saddr->sa_data); 4090 netif_addr_unlock_bh(netdev); 4091 4092 mlx5e_nic_set_rx_mode(priv); 4093 4094 return 0; 4095} 4096 4097#define MLX5E_SET_FEATURE(features, feature, enable) \ 4098 do { \ 4099 if (enable) \ 4100 *features |= feature; \ 4101 else \ 4102 *features &= ~feature; \ 4103 } while (0) 4104 4105typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 4106 4107static int set_feature_lro(struct net_device *netdev, bool enable) 4108{ 4109 struct mlx5e_priv *priv = netdev_priv(netdev); 4110 struct mlx5_core_dev *mdev = priv->mdev; 4111 struct mlx5e_params *cur_params; 4112 struct mlx5e_params new_params; 4113 bool reset = true; 4114 int err = 0; 4115 4116 mutex_lock(&priv->state_lock); 4117 4118 cur_params = &priv->channels.params; 4119 new_params = *cur_params; 4120 4121 if (enable) 4122 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO; 4123 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4124 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4125 else 4126 goto out; 4127 4128 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO && 4129 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) { 4130 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 4131 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) == 4132 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL)) 4133 reset = false; 4134 } 4135 } 4136 4137 err = mlx5e_safe_switch_params(priv, &new_params, 4138 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset); 4139out: 4140 mutex_unlock(&priv->state_lock); 4141 return err; 4142} 4143 4144static int set_feature_hw_gro(struct net_device *netdev, bool enable) 4145{ 4146 struct mlx5e_priv *priv = netdev_priv(netdev); 4147 struct mlx5e_params new_params; 4148 bool reset = true; 4149 int err = 0; 4150 4151 mutex_lock(&priv->state_lock); 4152 new_params = priv->channels.params; 4153 4154 if (enable) { 4155 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO; 4156 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 4157 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4158 } else { 4159 goto out; 4160 } 4161 4162 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 4163out: 4164 mutex_unlock(&priv->state_lock); 4165 return err; 4166} 4167 4168static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 4169{ 4170 struct mlx5e_priv *priv = netdev_priv(netdev); 4171 4172 if (enable) 4173 mlx5e_enable_cvlan_filter(priv->fs, 4174 !!(priv->netdev->flags & IFF_PROMISC)); 4175 else 4176 mlx5e_disable_cvlan_filter(priv->fs, 4177 !!(priv->netdev->flags & IFF_PROMISC)); 4178 4179 return 0; 4180} 4181 4182static int set_feature_hw_tc(struct net_device *netdev, bool enable) 4183{ 4184 struct mlx5e_priv *priv = netdev_priv(netdev); 4185 int err = 0; 4186 4187#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 4188 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : 4189 MLX5_TC_FLAG(NIC_OFFLOAD); 4190 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { 4191 netdev_err(netdev, 4192 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 4193 return -EINVAL; 4194 } 4195#endif 4196 4197 mutex_lock(&priv->state_lock); 4198 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { 4199 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); 4200 err = -EINVAL; 4201 } 4202 mutex_unlock(&priv->state_lock); 4203 4204 return err; 4205} 4206 4207static int set_feature_rx_all(struct net_device *netdev, bool enable) 4208{ 4209 struct mlx5e_priv *priv = netdev_priv(netdev); 4210 struct mlx5_core_dev *mdev = priv->mdev; 4211 4212 return mlx5_set_port_fcs(mdev, !enable); 4213} 4214 4215static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4216{ 4217 return (struct dim_cq_moder) { 4218 .cq_period_mode = cq_period_mode, 4219 .pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS, 4220 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 4221 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE : 4222 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC, 4223 }; 4224} 4225 4226bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 4227 bool dim_enabled) 4228{ 4229 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 4230 4231 if (dim_enabled) 4232 *cq_moder = net_dim_get_def_rx_moderation(cq_period_mode); 4233 else 4234 *cq_moder = mlx5e_get_def_rx_moderation(cq_period_mode); 4235 4236 return reset_needed; 4237} 4238 4239bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 4240 bool dim_enabled, bool keep_dim_state) 4241{ 4242 bool reset = false; 4243 int i; 4244 4245 for (i = 0; i < chs->num; i++) { 4246 if (keep_dim_state) 4247 dim_enabled = !!chs->c[i]->rq.dim; 4248 4249 reset |= mlx5e_reset_rx_moderation(&chs->c[i]->rx_cq_moder, 4250 cq_period_mode, dim_enabled); 4251 } 4252 4253 return reset; 4254} 4255 4256static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) 4257{ 4258 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {}; 4259 bool supported, curr_state; 4260 int err; 4261 4262 if (!MLX5_CAP_GEN(mdev, ports_check)) 4263 return 0; 4264 4265 err = mlx5_query_ports_check(mdev, in, sizeof(in)); 4266 if (err) 4267 return err; 4268 4269 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap); 4270 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc); 4271 4272 if (!supported || enable == curr_state) 4273 return 0; 4274 4275 MLX5_SET(pcmr_reg, in, local_port, 1); 4276 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable); 4277 4278 return mlx5_set_ports_check(mdev, in, sizeof(in)); 4279} 4280 4281static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx) 4282{ 4283 struct mlx5_core_dev *mdev = priv->mdev; 4284 bool enable = *(bool *)ctx; 4285 4286 return mlx5e_set_rx_port_ts(mdev, enable); 4287} 4288 4289static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 4290{ 4291 struct mlx5e_priv *priv = netdev_priv(netdev); 4292 struct mlx5e_channels *chs = &priv->channels; 4293 struct mlx5e_params new_params; 4294 int err; 4295 bool rx_ts_over_crc = !enable; 4296 4297 mutex_lock(&priv->state_lock); 4298 4299 new_params = chs->params; 4300 new_params.scatter_fcs_en = enable; 4301 err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap, 4302 &rx_ts_over_crc, true); 4303 mutex_unlock(&priv->state_lock); 4304 return err; 4305} 4306 4307static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 4308{ 4309 struct mlx5e_priv *priv = netdev_priv(netdev); 4310 int err = 0; 4311 4312 mutex_lock(&priv->state_lock); 4313 4314 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); 4315 priv->channels.params.vlan_strip_disable = !enable; 4316 4317 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4318 goto unlock; 4319 4320 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 4321 if (err) { 4322 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); 4323 priv->channels.params.vlan_strip_disable = enable; 4324 } 4325unlock: 4326 mutex_unlock(&priv->state_lock); 4327 4328 return err; 4329} 4330 4331int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 4332{ 4333 struct mlx5e_priv *priv = netdev_priv(dev); 4334 struct mlx5e_flow_steering *fs = priv->fs; 4335 4336 if (mlx5e_is_uplink_rep(priv)) 4337 return 0; /* no vlan table for uplink rep */ 4338 4339 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); 4340} 4341 4342int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 4343{ 4344 struct mlx5e_priv *priv = netdev_priv(dev); 4345 struct mlx5e_flow_steering *fs = priv->fs; 4346 4347 if (mlx5e_is_uplink_rep(priv)) 4348 return 0; /* no vlan table for uplink rep */ 4349 4350 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); 4351} 4352 4353#ifdef CONFIG_MLX5_EN_ARFS 4354static int set_feature_arfs(struct net_device *netdev, bool enable) 4355{ 4356 struct mlx5e_priv *priv = netdev_priv(netdev); 4357 int err; 4358 4359 if (enable) 4360 err = mlx5e_arfs_enable(priv->fs); 4361 else 4362 err = mlx5e_arfs_disable(priv->fs); 4363 4364 return err; 4365} 4366#endif 4367 4368static int mlx5e_handle_feature(struct net_device *netdev, 4369 netdev_features_t *features, 4370 netdev_features_t feature, 4371 mlx5e_feature_handler feature_handler) 4372{ 4373 netdev_features_t changes = *features ^ netdev->features; 4374 bool enable = !!(*features & feature); 4375 int err; 4376 4377 if (!(changes & feature)) 4378 return 0; 4379 4380 err = feature_handler(netdev, enable); 4381 if (err) { 4382 MLX5E_SET_FEATURE(features, feature, !enable); 4383 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 4384 enable ? "Enable" : "Disable", &feature, err); 4385 return err; 4386 } 4387 4388 return 0; 4389} 4390 4391void mlx5e_set_xdp_feature(struct mlx5e_priv *priv) 4392{ 4393 struct mlx5e_params *params = &priv->channels.params; 4394 struct net_device *netdev = priv->netdev; 4395 xdp_features_t val = 0; 4396 4397 if (netdev->netdev_ops->ndo_bpf && 4398 params->packet_merge.type == MLX5E_PACKET_MERGE_NONE) 4399 val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4400 NETDEV_XDP_ACT_XSK_ZEROCOPY | 4401 NETDEV_XDP_ACT_RX_SG; 4402 4403 if (netdev->netdev_ops->ndo_xdp_xmit && params->xdp_prog) 4404 val |= NETDEV_XDP_ACT_NDO_XMIT | 4405 NETDEV_XDP_ACT_NDO_XMIT_SG; 4406 4407 xdp_set_features_flag_locked(netdev, val); 4408} 4409 4410int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 4411{ 4412 netdev_features_t oper_features = features; 4413 int err = 0; 4414 4415#define MLX5E_HANDLE_FEATURE(feature, handler) \ 4416 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4417 4418 if (features & (NETIF_F_GRO_HW | NETIF_F_LRO)) { 4419 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4420 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4421 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4422 } else { 4423 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4424 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4425 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4426 } 4427 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4428 set_feature_cvlan_filter); 4429 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4430 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4431 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4432#ifdef CONFIG_MLX5_EN_ARFS 4433 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 4434#endif 4435 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); 4436 4437 if (err) { 4438 netdev->features = oper_features; 4439 return -EINVAL; 4440 } 4441 4442 return 0; 4443} 4444 4445static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev, 4446 netdev_features_t features) 4447{ 4448 features &= ~NETIF_F_HW_TLS_RX; 4449 if (netdev->features & NETIF_F_HW_TLS_RX) 4450 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); 4451 4452 features &= ~NETIF_F_HW_TLS_TX; 4453 if (netdev->features & NETIF_F_HW_TLS_TX) 4454 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); 4455 4456 features &= ~NETIF_F_NTUPLE; 4457 if (netdev->features & NETIF_F_NTUPLE) 4458 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n"); 4459 4460 features &= ~NETIF_F_GRO_HW; 4461 if (netdev->features & NETIF_F_GRO_HW) 4462 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); 4463 4464 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4465 if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) 4466 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); 4467 4468 features &= ~NETIF_F_HW_MACSEC; 4469 if (netdev->features & NETIF_F_HW_MACSEC) 4470 netdev_warn(netdev, "Disabling HW MACsec offload, not supported in switchdev mode\n"); 4471 4472 return features; 4473} 4474 4475static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 4476 netdev_features_t features) 4477{ 4478 struct netdev_config *cfg = netdev->cfg_pending; 4479 struct mlx5e_priv *priv = netdev_priv(netdev); 4480 struct mlx5e_vlan_table *vlan; 4481 struct mlx5e_params *params; 4482 4483 if (!netif_device_present(netdev)) 4484 return features; 4485 4486 vlan = mlx5e_fs_get_vlan(priv->fs); 4487 mutex_lock(&priv->state_lock); 4488 params = &priv->channels.params; 4489 if (!vlan || 4490 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { 4491 /* HW strips the outer C-tag header, this is a problem 4492 * for S-tag traffic. 4493 */ 4494 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4495 if (!params->vlan_strip_disable) 4496 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 4497 } 4498 4499 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4500 if (features & NETIF_F_LRO) { 4501 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 4502 features &= ~NETIF_F_LRO; 4503 } 4504 if (features & NETIF_F_GRO_HW) { 4505 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n"); 4506 features &= ~NETIF_F_GRO_HW; 4507 } 4508 } 4509 4510 if (params->xdp_prog) { 4511 if (features & NETIF_F_LRO) { 4512 netdev_warn(netdev, "LRO is incompatible with XDP\n"); 4513 features &= ~NETIF_F_LRO; 4514 } 4515 if (features & NETIF_F_GRO_HW) { 4516 netdev_warn(netdev, "HW GRO is incompatible with XDP\n"); 4517 features &= ~NETIF_F_GRO_HW; 4518 } 4519 } 4520 4521 if (priv->xsk.refcnt) { 4522 if (features & NETIF_F_LRO) { 4523 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", 4524 priv->xsk.refcnt); 4525 features &= ~NETIF_F_LRO; 4526 } 4527 if (features & NETIF_F_GRO_HW) { 4528 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", 4529 priv->xsk.refcnt); 4530 features &= ~NETIF_F_GRO_HW; 4531 } 4532 } 4533 4534 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 4535 features &= ~NETIF_F_RXHASH; 4536 if (netdev->features & NETIF_F_RXHASH) 4537 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 4538 4539 if (features & NETIF_F_GRO_HW) { 4540 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n"); 4541 features &= ~NETIF_F_GRO_HW; 4542 } 4543 } 4544 4545 /* The header-data split ring param requires HW GRO to stay enabled. */ 4546 if (cfg && cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 4547 !(features & NETIF_F_GRO_HW)) { 4548 netdev_warn(netdev, "Keeping HW-GRO enabled, TCP header-data split depends on it\n"); 4549 features |= NETIF_F_GRO_HW; 4550 } 4551 4552 if (mlx5e_is_uplink_rep(priv)) { 4553 features = mlx5e_fix_uplink_rep_features(netdev, features); 4554 netdev->netns_immutable = true; 4555 } else { 4556 netdev->netns_immutable = false; 4557 } 4558 4559 mutex_unlock(&priv->state_lock); 4560 4561 return features; 4562} 4563 4564static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 4565 struct mlx5e_channels *chs, 4566 struct mlx5e_params *new_params, 4567 struct mlx5_core_dev *mdev) 4568{ 4569 u16 ix; 4570 4571 for (ix = 0; ix < chs->params.num_channels; ix++) { 4572 struct xsk_buff_pool *xsk_pool = 4573 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); 4574 struct mlx5e_xsk_param xsk; 4575 int max_xdp_mtu; 4576 4577 if (!xsk_pool) 4578 continue; 4579 4580 mlx5e_build_xsk_param(xsk_pool, &xsk); 4581 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk); 4582 4583 /* Validate XSK params and XDP MTU in advance */ 4584 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) || 4585 new_params->sw_mtu > max_xdp_mtu) { 4586 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); 4587 int max_mtu_frame, max_mtu_page, max_mtu; 4588 4589 /* Two criteria must be met: 4590 * 1. HW MTU + all headrooms <= XSK frame size. 4591 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 4592 */ 4593 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 4594 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); 4595 max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu); 4596 4597 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n", 4598 new_params->sw_mtu, ix, max_mtu); 4599 return false; 4600 } 4601 } 4602 4603 return true; 4604} 4605 4606static bool mlx5e_params_validate_xdp(struct net_device *netdev, 4607 struct mlx5_core_dev *mdev, 4608 struct mlx5e_params *params) 4609{ 4610 bool is_linear; 4611 4612 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4613 * the XDP program. 4614 */ 4615 is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ? 4616 mlx5e_rx_is_linear_skb(mdev, params, NULL) : 4617 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL); 4618 4619 if (!is_linear) { 4620 if (!params->xdp_prog->aux->xdp_has_frags) { 4621 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n", 4622 params->sw_mtu, 4623 mlx5e_xdp_max_mtu(params, NULL)); 4624 return false; 4625 } 4626 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4627 !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) { 4628 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", 4629 params->sw_mtu, 4630 mlx5e_xdp_max_mtu(params, NULL)); 4631 return false; 4632 } 4633 } 4634 4635 return true; 4636} 4637 4638int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 4639 mlx5e_fp_preactivate preactivate) 4640{ 4641 struct mlx5e_priv *priv = netdev_priv(netdev); 4642 struct mlx5e_params new_params; 4643 struct mlx5e_params *params; 4644 bool reset = true; 4645 int err = 0; 4646 4647 mutex_lock(&priv->state_lock); 4648 4649 params = &priv->channels.params; 4650 4651 new_params = *params; 4652 new_params.sw_mtu = new_mtu; 4653 err = mlx5e_validate_params(priv->mdev, &new_params); 4654 if (err) 4655 goto out; 4656 4657 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, 4658 &new_params)) { 4659 err = -EINVAL; 4660 goto out; 4661 } 4662 4663 if (priv->xsk.refcnt && 4664 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 4665 &new_params, priv->mdev)) { 4666 err = -EINVAL; 4667 goto out; 4668 } 4669 4670 if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4671 reset = false; 4672 4673 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4674 params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) { 4675 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL); 4676 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, 4677 &new_params, NULL); 4678 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL); 4679 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL); 4680 4681 /* Always reset in linear mode - hw_mtu is used in data path. 4682 * Check that the mode was non-linear and didn't change. 4683 * If XSK is active, XSK RQs are linear. 4684 * Reset if the RQ size changed, even if it's non-linear. 4685 */ 4686 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt && 4687 sz_old == sz_new) 4688 reset = false; 4689 } 4690 4691 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset); 4692 4693out: 4694 WRITE_ONCE(netdev->mtu, params->sw_mtu); 4695 mutex_unlock(&priv->state_lock); 4696 4697 if (!err) 4698 netdev_update_features(netdev); 4699 4700 return err; 4701} 4702 4703static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 4704{ 4705 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4706} 4707 4708int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) 4709{ 4710 bool set = *(bool *)ctx; 4711 4712 return mlx5e_ptp_rx_manage_fs(priv, set); 4713} 4714 4715static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) 4716{ 4717 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4718 int err; 4719 4720 if (!rx_filter) 4721 /* Reset CQE compression to Admin default */ 4722 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false); 4723 4724 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4725 return 0; 4726 4727 /* Disable CQE compression */ 4728 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4729 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true); 4730 if (err) 4731 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4732 4733 return err; 4734} 4735 4736static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) 4737{ 4738 struct mlx5e_params new_params; 4739 4740 if (ptp_rx == priv->channels.params.ptp_rx) 4741 return 0; 4742 4743 new_params = priv->channels.params; 4744 new_params.ptp_rx = ptp_rx; 4745 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, 4746 &new_params.ptp_rx, true); 4747} 4748 4749int mlx5e_hwtstamp_set(struct mlx5e_priv *priv, 4750 struct kernel_hwtstamp_config *config, 4751 struct netlink_ext_ack *extack) 4752{ 4753 bool rx_cqe_compress_def; 4754 bool ptp_rx; 4755 int err; 4756 4757 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4758 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) { 4759 NL_SET_ERR_MSG_MOD(extack, 4760 "Timestamps are not supported on this device"); 4761 return -EOPNOTSUPP; 4762 } 4763 4764 /* TX HW timestamp */ 4765 switch (config->tx_type) { 4766 case HWTSTAMP_TX_OFF: 4767 case HWTSTAMP_TX_ON: 4768 break; 4769 default: 4770 return -ERANGE; 4771 } 4772 4773 mutex_lock(&priv->state_lock); 4774 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4775 4776 /* RX HW timestamp */ 4777 switch (config->rx_filter) { 4778 case HWTSTAMP_FILTER_NONE: 4779 ptp_rx = false; 4780 break; 4781 case HWTSTAMP_FILTER_ALL: 4782 case HWTSTAMP_FILTER_SOME: 4783 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4784 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4785 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4786 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4787 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4788 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4789 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4790 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4791 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4792 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4793 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4794 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4795 case HWTSTAMP_FILTER_NTP_ALL: 4796 config->rx_filter = HWTSTAMP_FILTER_ALL; 4797 /* ptp_rx is set if both HW TS is set and CQE 4798 * compression is set 4799 */ 4800 ptp_rx = rx_cqe_compress_def; 4801 break; 4802 default: 4803 err = -ERANGE; 4804 goto err_unlock; 4805 } 4806 4807 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) 4808 err = mlx5e_hwstamp_config_no_ptp_rx(priv, 4809 config->rx_filter != HWTSTAMP_FILTER_NONE); 4810 else 4811 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); 4812 if (err) 4813 goto err_unlock; 4814 4815 priv->hwtstamp_config = *config; 4816 mutex_unlock(&priv->state_lock); 4817 4818 /* might need to fix some features */ 4819 netdev_update_features(priv->netdev); 4820 4821 return 0; 4822err_unlock: 4823 mutex_unlock(&priv->state_lock); 4824 return err; 4825} 4826 4827static int mlx5e_hwtstamp_set_ndo(struct net_device *netdev, 4828 struct kernel_hwtstamp_config *config, 4829 struct netlink_ext_ack *extack) 4830{ 4831 struct mlx5e_priv *priv = netdev_priv(netdev); 4832 4833 return mlx5e_hwtstamp_set(priv, config, extack); 4834} 4835 4836int mlx5e_hwtstamp_get(struct mlx5e_priv *priv, 4837 struct kernel_hwtstamp_config *config) 4838{ 4839 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4840 return -EOPNOTSUPP; 4841 4842 *config = priv->hwtstamp_config; 4843 4844 return 0; 4845} 4846 4847static int mlx5e_hwtstamp_get_ndo(struct net_device *dev, 4848 struct kernel_hwtstamp_config *config) 4849{ 4850 struct mlx5e_priv *priv = netdev_priv(dev); 4851 4852 return mlx5e_hwtstamp_get(priv, config); 4853} 4854 4855#ifdef CONFIG_MLX5_ESWITCH 4856int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4857{ 4858 struct mlx5e_priv *priv = netdev_priv(dev); 4859 struct mlx5_core_dev *mdev = priv->mdev; 4860 4861 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4862} 4863 4864static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4865 __be16 vlan_proto) 4866{ 4867 struct mlx5e_priv *priv = netdev_priv(dev); 4868 struct mlx5_core_dev *mdev = priv->mdev; 4869 4870 if (vlan_proto != htons(ETH_P_8021Q)) 4871 return -EPROTONOSUPPORT; 4872 4873 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4874 vlan, qos); 4875} 4876 4877static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4878{ 4879 struct mlx5e_priv *priv = netdev_priv(dev); 4880 struct mlx5_core_dev *mdev = priv->mdev; 4881 4882 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4883} 4884 4885static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4886{ 4887 struct mlx5e_priv *priv = netdev_priv(dev); 4888 struct mlx5_core_dev *mdev = priv->mdev; 4889 4890 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4891} 4892 4893int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4894 int max_tx_rate) 4895{ 4896 struct mlx5e_priv *priv = netdev_priv(dev); 4897 struct mlx5_core_dev *mdev = priv->mdev; 4898 4899 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4900 max_tx_rate, min_tx_rate); 4901} 4902 4903static int mlx5_vport_link2ifla(u8 esw_link) 4904{ 4905 switch (esw_link) { 4906 case MLX5_VPORT_ADMIN_STATE_DOWN: 4907 return IFLA_VF_LINK_STATE_DISABLE; 4908 case MLX5_VPORT_ADMIN_STATE_UP: 4909 return IFLA_VF_LINK_STATE_ENABLE; 4910 } 4911 return IFLA_VF_LINK_STATE_AUTO; 4912} 4913 4914static int mlx5_ifla_link2vport(u8 ifla_link) 4915{ 4916 switch (ifla_link) { 4917 case IFLA_VF_LINK_STATE_DISABLE: 4918 return MLX5_VPORT_ADMIN_STATE_DOWN; 4919 case IFLA_VF_LINK_STATE_ENABLE: 4920 return MLX5_VPORT_ADMIN_STATE_UP; 4921 } 4922 return MLX5_VPORT_ADMIN_STATE_AUTO; 4923} 4924 4925static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4926 int link_state) 4927{ 4928 struct mlx5e_priv *priv = netdev_priv(dev); 4929 struct mlx5_core_dev *mdev = priv->mdev; 4930 4931 if (mlx5e_is_uplink_rep(priv)) 4932 return -EOPNOTSUPP; 4933 4934 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4935 mlx5_ifla_link2vport(link_state)); 4936} 4937 4938int mlx5e_get_vf_config(struct net_device *dev, 4939 int vf, struct ifla_vf_info *ivi) 4940{ 4941 struct mlx5e_priv *priv = netdev_priv(dev); 4942 struct mlx5_core_dev *mdev = priv->mdev; 4943 int err; 4944 4945 if (!netif_device_present(dev)) 4946 return -EOPNOTSUPP; 4947 4948 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 4949 if (err) 4950 return err; 4951 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 4952 return 0; 4953} 4954 4955int mlx5e_get_vf_stats(struct net_device *dev, 4956 int vf, struct ifla_vf_stats *vf_stats) 4957{ 4958 struct mlx5e_priv *priv = netdev_priv(dev); 4959 struct mlx5_core_dev *mdev = priv->mdev; 4960 4961 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 4962 vf_stats); 4963} 4964 4965static bool 4966mlx5e_has_offload_stats(const struct net_device *dev, int attr_id) 4967{ 4968 struct mlx5e_priv *priv = netdev_priv(dev); 4969 4970 if (!netif_device_present(dev)) 4971 return false; 4972 4973 if (!mlx5e_is_uplink_rep(priv)) 4974 return false; 4975 4976 return mlx5e_rep_has_offload_stats(dev, attr_id); 4977} 4978 4979static int 4980mlx5e_get_offload_stats(int attr_id, const struct net_device *dev, 4981 void *sp) 4982{ 4983 struct mlx5e_priv *priv = netdev_priv(dev); 4984 4985 if (!mlx5e_is_uplink_rep(priv)) 4986 return -EOPNOTSUPP; 4987 4988 return mlx5e_rep_get_offload_stats(attr_id, dev, sp); 4989} 4990#endif 4991 4992static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type) 4993{ 4994 switch (proto_type) { 4995 case IPPROTO_GRE: 4996 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre); 4997 case IPPROTO_IPIP: 4998 case IPPROTO_IPV6: 4999 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) || 5000 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx)); 5001 default: 5002 return false; 5003 } 5004} 5005 5006static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev, 5007 struct sk_buff *skb) 5008{ 5009 switch (skb->inner_protocol) { 5010 case htons(ETH_P_IP): 5011 case htons(ETH_P_IPV6): 5012 case htons(ETH_P_TEB): 5013 return true; 5014 case htons(ETH_P_MPLS_UC): 5015 case htons(ETH_P_MPLS_MC): 5016 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre); 5017 } 5018 return false; 5019} 5020 5021static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 5022 struct sk_buff *skb, 5023 netdev_features_t features) 5024{ 5025 unsigned int offset = 0; 5026 struct udphdr *udph; 5027 u8 proto; 5028 u16 port; 5029 5030 switch (vlan_get_protocol(skb)) { 5031 case htons(ETH_P_IP): 5032 proto = ip_hdr(skb)->protocol; 5033 break; 5034 case htons(ETH_P_IPV6): 5035 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 5036 break; 5037 default: 5038 goto out; 5039 } 5040 5041 switch (proto) { 5042 case IPPROTO_GRE: 5043 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb)) 5044 return features; 5045 break; 5046 case IPPROTO_IPIP: 5047 case IPPROTO_IPV6: 5048 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP)) 5049 return features; 5050 break; 5051 case IPPROTO_UDP: 5052 udph = udp_hdr(skb); 5053 port = be16_to_cpu(udph->dest); 5054 5055 /* Verify if UDP port is being offloaded by HW */ 5056 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 5057 return vxlan_features_check(skb, features); 5058 5059#if IS_ENABLED(CONFIG_GENEVE) 5060 /* Support Geneve offload for default UDP port */ 5061 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 5062 return features; 5063#endif 5064 break; 5065#ifdef CONFIG_MLX5_EN_IPSEC 5066 case IPPROTO_ESP: 5067 return mlx5e_ipsec_feature_check(skb, features); 5068#endif 5069 } 5070 5071out: 5072 /* Disable CSUM and GSO if skb cannot be offloaded by HW */ 5073 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5074} 5075 5076netdev_features_t mlx5e_features_check(struct sk_buff *skb, 5077 struct net_device *netdev, 5078 netdev_features_t features) 5079{ 5080 struct mlx5e_priv *priv = netdev_priv(netdev); 5081 5082 features = vlan_features_check(skb, features); 5083 5084 /* Validate if the tunneled packet is being offloaded by HW */ 5085 if (skb->encapsulation && 5086 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 5087 return mlx5e_tunnel_features_check(priv, skb, features); 5088 5089 return features; 5090} 5091 5092static void mlx5e_tx_timeout_work(struct work_struct *work) 5093{ 5094 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 5095 tx_timeout_work); 5096 struct net_device *netdev = priv->netdev; 5097 int i; 5098 5099 for (i = 0; i < netdev->real_num_tx_queues; i++) { 5100 struct netdev_queue *dev_queue = 5101 netdev_get_tx_queue(netdev, i); 5102 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 5103 5104 if (!netif_xmit_stopped(dev_queue)) 5105 continue; 5106 5107 if (mlx5e_reporter_tx_timeout(sq)) 5108 /* break if tried to reopened channels */ 5109 break; 5110 } 5111} 5112 5113static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 5114{ 5115 struct mlx5e_priv *priv = netdev_priv(dev); 5116 5117 netdev_err(dev, "TX timeout detected\n"); 5118 queue_work(priv->wq, &priv->tx_timeout_work); 5119} 5120 5121static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev, 5122 struct mlx5e_params *params) 5123{ 5124 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 5125 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n"); 5126 return -EINVAL; 5127 } 5128 5129 if (!mlx5e_params_validate_xdp(netdev, mdev, params)) 5130 return -EINVAL; 5131 5132 return 0; 5133} 5134 5135static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog) 5136{ 5137 struct bpf_prog *old_prog; 5138 5139 old_prog = rcu_replace_pointer(rq->xdp_prog, prog, 5140 lockdep_is_held(&rq->priv->state_lock)); 5141 if (old_prog) 5142 bpf_prog_put(old_prog); 5143} 5144 5145static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 5146{ 5147 struct mlx5e_priv *priv = netdev_priv(netdev); 5148 struct mlx5e_params new_params; 5149 struct bpf_prog *old_prog; 5150 int err = 0; 5151 bool reset; 5152 int i; 5153 5154 mutex_lock(&priv->state_lock); 5155 5156 new_params = priv->channels.params; 5157 new_params.xdp_prog = prog; 5158 5159 if (prog) { 5160 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params); 5161 if (err) 5162 goto unlock; 5163 } 5164 5165 /* no need for full reset when exchanging programs */ 5166 reset = (!priv->channels.params.xdp_prog || !prog); 5167 5168 old_prog = priv->channels.params.xdp_prog; 5169 5170 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 5171 if (err) 5172 goto unlock; 5173 5174 if (old_prog) 5175 bpf_prog_put(old_prog); 5176 5177 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 5178 goto unlock; 5179 5180 /* exchanging programs w/o reset, we update ref counts on behalf 5181 * of the channels RQs here. 5182 */ 5183 bpf_prog_add(prog, priv->channels.num); 5184 for (i = 0; i < priv->channels.num; i++) { 5185 struct mlx5e_channel *c = priv->channels.c[i]; 5186 5187 mlx5e_rq_replace_xdp_prog(&c->rq, prog); 5188 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) { 5189 bpf_prog_inc(prog); 5190 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog); 5191 } 5192 } 5193 5194unlock: 5195 mutex_unlock(&priv->state_lock); 5196 5197 /* Need to fix some features. */ 5198 if (!err) 5199 netdev_update_features(netdev); 5200 5201 return err; 5202} 5203 5204static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 5205{ 5206 switch (xdp->command) { 5207 case XDP_SETUP_PROG: 5208 return mlx5e_xdp_set(dev, xdp->prog); 5209 case XDP_SETUP_XSK_POOL: 5210 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool, 5211 xdp->xsk.queue_id); 5212 default: 5213 return -EINVAL; 5214 } 5215} 5216 5217#ifdef CONFIG_MLX5_ESWITCH 5218static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 5219 struct net_device *dev, u32 filter_mask, 5220 int nlflags) 5221{ 5222 struct mlx5e_priv *priv = netdev_priv(dev); 5223 struct mlx5_core_dev *mdev = priv->mdev; 5224 u8 mode, setting; 5225 5226 if (mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting)) 5227 return -EOPNOTSUPP; 5228 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 5229 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 5230 mode, 5231 0, 0, nlflags, filter_mask, NULL); 5232} 5233 5234static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 5235 u16 flags, struct netlink_ext_ack *extack) 5236{ 5237 struct mlx5e_priv *priv = netdev_priv(dev); 5238 struct mlx5_core_dev *mdev = priv->mdev; 5239 struct nlattr *attr, *br_spec; 5240 u16 mode = BRIDGE_MODE_UNDEF; 5241 u8 setting; 5242 int rem; 5243 5244 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 5245 if (!br_spec) 5246 return -EINVAL; 5247 5248 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 5249 mode = nla_get_u16(attr); 5250 if (mode > BRIDGE_MODE_VEPA) 5251 return -EINVAL; 5252 5253 break; 5254 } 5255 5256 if (mode == BRIDGE_MODE_UNDEF) 5257 return -EINVAL; 5258 5259 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 5260 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 5261} 5262#endif 5263 5264const struct net_device_ops mlx5e_netdev_ops = { 5265 .ndo_open = mlx5e_open, 5266 .ndo_stop = mlx5e_close, 5267 .ndo_start_xmit = mlx5e_xmit, 5268 .ndo_setup_tc = mlx5e_setup_tc, 5269 .ndo_select_queue = mlx5e_select_queue, 5270 .ndo_get_stats64 = mlx5e_get_stats, 5271 .ndo_set_rx_mode = mlx5e_set_rx_mode, 5272 .ndo_set_mac_address = mlx5e_set_mac, 5273 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 5274 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 5275 .ndo_set_features = mlx5e_set_features, 5276 .ndo_fix_features = mlx5e_fix_features, 5277 .ndo_change_mtu = mlx5e_change_nic_mtu, 5278 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 5279 .ndo_features_check = mlx5e_features_check, 5280 .ndo_tx_timeout = mlx5e_tx_timeout, 5281 .ndo_bpf = mlx5e_xdp, 5282 .ndo_xdp_xmit = mlx5e_xdp_xmit, 5283 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 5284 .ndo_hwtstamp_get = mlx5e_hwtstamp_get_ndo, 5285 .ndo_hwtstamp_set = mlx5e_hwtstamp_set_ndo, 5286#ifdef CONFIG_MLX5_EN_ARFS 5287 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 5288#endif 5289#ifdef CONFIG_MLX5_ESWITCH 5290 .ndo_bridge_setlink = mlx5e_bridge_setlink, 5291 .ndo_bridge_getlink = mlx5e_bridge_getlink, 5292 5293 /* SRIOV E-Switch NDOs */ 5294 .ndo_set_vf_mac = mlx5e_set_vf_mac, 5295 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 5296 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 5297 .ndo_set_vf_trust = mlx5e_set_vf_trust, 5298 .ndo_set_vf_rate = mlx5e_set_vf_rate, 5299 .ndo_get_vf_config = mlx5e_get_vf_config, 5300 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 5301 .ndo_get_vf_stats = mlx5e_get_vf_stats, 5302 .ndo_has_offload_stats = mlx5e_has_offload_stats, 5303 .ndo_get_offload_stats = mlx5e_get_offload_stats, 5304#endif 5305}; 5306 5307void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu) 5308{ 5309 struct mlx5e_params *params = &priv->channels.params; 5310 struct mlx5_core_dev *mdev = priv->mdev; 5311 5312 params->sw_mtu = mtu; 5313 params->hard_mtu = MLX5E_ETH_HARD_MTU; 5314 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 5315 priv->max_nch); 5316 mlx5e_params_mqprio_reset(params); 5317 5318 /* SQ */ 5319 params->log_sq_size = is_kdump_kernel() ? 5320 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 5321 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 5322 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5323 5324 /* XDP SQ */ 5325 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5326 5327 /* set CQE compression */ 5328 params->rx_cqe_compress_def = false; 5329 if (MLX5_CAP_GEN(mdev, cqe_compression) && 5330 MLX5_CAP_GEN(mdev, vport_group_manager)) 5331 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 5332 5333 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 5334 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 5335 5336 /* RQ */ 5337 mlx5e_build_rq_params(mdev, params); 5338 5339 params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev); 5340 5341 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 5342 5343 /* CQ moderation params */ 5344 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5345 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5346 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5347 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5348 params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe); 5349 params->tx_moder_use_cqe_mode = false; 5350 mlx5e_reset_rx_moderation(¶ms->rx_cq_moderation, params->rx_moder_use_cqe_mode, 5351 params->rx_dim_enabled); 5352 mlx5e_reset_tx_moderation(¶ms->tx_cq_moderation, params->tx_moder_use_cqe_mode, 5353 params->tx_dim_enabled); 5354 5355 /* TX inline */ 5356 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 5357 5358 /* AF_XDP */ 5359 params->xsk = xsk; 5360 5361 /* Do not update netdev->features directly in here 5362 * on mlx5e_attach_netdev() we will call mlx5e_update_features() 5363 * To update netdev->features please modify mlx5e_fix_features() 5364 */ 5365} 5366 5367static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 5368{ 5369 struct mlx5e_priv *priv = netdev_priv(netdev); 5370 u8 addr[ETH_ALEN]; 5371 5372 mlx5_query_mac_address(priv->mdev, addr); 5373 if (is_zero_ether_addr(addr) && 5374 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 5375 eth_hw_addr_random(netdev); 5376 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 5377 return; 5378 } 5379 5380 eth_hw_addr_set(netdev, addr); 5381} 5382 5383static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, 5384 unsigned int entry, struct udp_tunnel_info *ti) 5385{ 5386 struct mlx5e_priv *priv = netdev_priv(netdev); 5387 5388 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); 5389} 5390 5391static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, 5392 unsigned int entry, struct udp_tunnel_info *ti) 5393{ 5394 struct mlx5e_priv *priv = netdev_priv(netdev); 5395 5396 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); 5397} 5398 5399void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) 5400{ 5401 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 5402 return; 5403 5404 priv->nic_info.set_port = mlx5e_vxlan_set_port; 5405 priv->nic_info.unset_port = mlx5e_vxlan_unset_port; 5406 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; 5407 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 5408 /* Don't count the space hard-coded to the IANA port */ 5409 priv->nic_info.tables[0].n_entries = 5410 mlx5_vxlan_max_udp_ports(priv->mdev) - 1; 5411 5412 priv->netdev->udp_tunnel_nic_info = &priv->nic_info; 5413} 5414 5415static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev) 5416{ 5417 int tt; 5418 5419 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { 5420 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt))) 5421 return true; 5422 } 5423 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)); 5424} 5425 5426static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i, 5427 struct netdev_queue_stats_rx *stats) 5428{ 5429 struct mlx5e_priv *priv = netdev_priv(dev); 5430 struct mlx5e_channel_stats *channel_stats; 5431 struct mlx5e_rq_stats *xskrq_stats; 5432 struct mlx5e_rq_stats *rq_stats; 5433 5434 if (mlx5e_is_uplink_rep(priv) || !priv->stats_nch) 5435 return; 5436 5437 channel_stats = priv->channel_stats[i]; 5438 xskrq_stats = &channel_stats->xskrq; 5439 rq_stats = &channel_stats->rq; 5440 5441 stats->packets = rq_stats->packets + xskrq_stats->packets; 5442 stats->bytes = rq_stats->bytes + xskrq_stats->bytes; 5443 stats->alloc_fail = rq_stats->buff_alloc_err + 5444 xskrq_stats->buff_alloc_err; 5445} 5446 5447static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i, 5448 struct netdev_queue_stats_tx *stats) 5449{ 5450 struct mlx5e_priv *priv = netdev_priv(dev); 5451 struct mlx5e_sq_stats *sq_stats; 5452 5453 if (!priv->stats_nch) 5454 return; 5455 5456 /* no special case needed for ptp htb etc since txq2sq_stats is kept up 5457 * to date for active sq_stats, otherwise get_base_stats takes care of 5458 * inactive sqs. 5459 */ 5460 sq_stats = priv->txq2sq_stats[i]; 5461 stats->packets = sq_stats->packets; 5462 stats->bytes = sq_stats->bytes; 5463} 5464 5465static void mlx5e_get_base_stats(struct net_device *dev, 5466 struct netdev_queue_stats_rx *rx, 5467 struct netdev_queue_stats_tx *tx) 5468{ 5469 struct mlx5e_priv *priv = netdev_priv(dev); 5470 struct mlx5e_ptp *ptp_channel; 5471 int i, tc; 5472 5473 if (!mlx5e_is_uplink_rep(priv)) { 5474 rx->packets = 0; 5475 rx->bytes = 0; 5476 rx->alloc_fail = 0; 5477 5478 for (i = priv->channels.params.num_channels; i < priv->stats_nch; i++) { 5479 struct netdev_queue_stats_rx rx_i = {0}; 5480 5481 mlx5e_get_queue_stats_rx(dev, i, &rx_i); 5482 5483 rx->packets += rx_i.packets; 5484 rx->bytes += rx_i.bytes; 5485 rx->alloc_fail += rx_i.alloc_fail; 5486 } 5487 5488 /* always report PTP RX stats from base as there is no 5489 * corresponding channel to report them under in 5490 * mlx5e_get_queue_stats_rx. 5491 */ 5492 if (priv->rx_ptp_opened) { 5493 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 5494 5495 rx->packets += rq_stats->packets; 5496 rx->bytes += rq_stats->bytes; 5497 } 5498 } 5499 5500 tx->packets = 0; 5501 tx->bytes = 0; 5502 5503 for (i = 0; i < priv->stats_nch; i++) { 5504 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 5505 5506 /* handle two cases: 5507 * 5508 * 1. channels which are active. In this case, 5509 * report only deactivated TCs on these channels. 5510 * 5511 * 2. channels which were deactivated 5512 * (i > priv->channels.params.num_channels) 5513 * must have all of their TCs [0 .. priv->max_opened_tc) 5514 * examined because deactivated channels will not be in the 5515 * range of [0..real_num_tx_queues) and will not have their 5516 * stats reported by mlx5e_get_queue_stats_tx. 5517 */ 5518 if (i < priv->channels.params.num_channels) 5519 tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 5520 else 5521 tc = 0; 5522 5523 for (; tc < priv->max_opened_tc; tc++) { 5524 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[tc]; 5525 5526 tx->packets += sq_stats->packets; 5527 tx->bytes += sq_stats->bytes; 5528 } 5529 } 5530 5531 /* if PTP TX was opened at some point and has since either: 5532 * - been shutdown and set to NULL, or 5533 * - simply disabled (bit unset) 5534 * 5535 * report stats directly from the ptp_stats structures as these queues 5536 * are now unavailable and there is no txq index to retrieve these 5537 * stats via calls to mlx5e_get_queue_stats_tx. 5538 */ 5539 ptp_channel = priv->channels.ptp; 5540 if (priv->tx_ptp_opened && (!ptp_channel || !test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state))) { 5541 for (tc = 0; tc < priv->max_opened_tc; tc++) { 5542 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[tc]; 5543 5544 tx->packets += sq_stats->packets; 5545 tx->bytes += sq_stats->bytes; 5546 } 5547 } 5548} 5549 5550static const struct netdev_stat_ops mlx5e_stat_ops = { 5551 .get_queue_stats_rx = mlx5e_get_queue_stats_rx, 5552 .get_queue_stats_tx = mlx5e_get_queue_stats_tx, 5553 .get_base_stats = mlx5e_get_base_stats, 5554}; 5555 5556struct mlx5_qmgmt_data { 5557 struct mlx5e_channel *c; 5558 struct mlx5e_channel_param cparam; 5559}; 5560 5561static int mlx5e_queue_mem_alloc(struct net_device *dev, void *newq, 5562 int queue_index) 5563{ 5564 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5565 struct mlx5e_priv *priv = netdev_priv(dev); 5566 struct mlx5e_channels *chs = &priv->channels; 5567 struct mlx5e_params params = chs->params; 5568 struct mlx5_core_dev *mdev; 5569 int err; 5570 5571 mutex_lock(&priv->state_lock); 5572 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 5573 err = -ENODEV; 5574 goto unlock; 5575 } 5576 5577 if (queue_index >= chs->num) { 5578 err = -ERANGE; 5579 goto unlock; 5580 } 5581 5582 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || 5583 chs->params.ptp_rx || 5584 chs->params.xdp_prog || 5585 priv->htb) { 5586 netdev_err(priv->netdev, 5587 "Cloning channels with Port/rx PTP, XDP or HTB is not supported\n"); 5588 err = -EOPNOTSUPP; 5589 goto unlock; 5590 } 5591 5592 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index); 5593 err = mlx5e_build_channel_param(mdev, ¶ms, &new->cparam); 5594 if (err) 5595 goto unlock; 5596 5597 err = mlx5e_open_channel(priv, queue_index, ¶ms, NULL, &new->c); 5598unlock: 5599 mutex_unlock(&priv->state_lock); 5600 return err; 5601} 5602 5603static void mlx5e_queue_mem_free(struct net_device *dev, void *mem) 5604{ 5605 struct mlx5_qmgmt_data *data = (struct mlx5_qmgmt_data *)mem; 5606 5607 /* not supposed to happen since mlx5e_queue_start never fails 5608 * but this is how this should be implemented just in case 5609 */ 5610 if (data->c) 5611 mlx5e_close_channel(data->c); 5612} 5613 5614static int mlx5e_queue_stop(struct net_device *dev, void *oldq, int queue_index) 5615{ 5616 /* In mlx5 a txq cannot be simply stopped in isolation, only restarted. 5617 * mlx5e_queue_start does not fail, we stop the old queue there. 5618 * TODO: Improve this. 5619 */ 5620 return 0; 5621} 5622 5623static int mlx5e_queue_start(struct net_device *dev, void *newq, 5624 int queue_index) 5625{ 5626 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5627 struct mlx5e_priv *priv = netdev_priv(dev); 5628 struct mlx5e_channel *old; 5629 5630 mutex_lock(&priv->state_lock); 5631 5632 /* stop and close the old */ 5633 old = priv->channels.c[queue_index]; 5634 mlx5e_deactivate_priv_channels(priv); 5635 /* close old before activating new, to avoid napi conflict */ 5636 mlx5e_close_channel(old); 5637 5638 /* start the new */ 5639 priv->channels.c[queue_index] = new->c; 5640 mlx5e_activate_priv_channels(priv); 5641 mutex_unlock(&priv->state_lock); 5642 return 0; 5643} 5644 5645static struct device *mlx5e_queue_get_dma_dev(struct net_device *dev, 5646 int queue_index) 5647{ 5648 struct mlx5e_priv *priv = netdev_priv(dev); 5649 struct mlx5e_channels *channels; 5650 struct device *pdev = NULL; 5651 struct mlx5e_channel *ch; 5652 5653 channels = &priv->channels; 5654 5655 mutex_lock(&priv->state_lock); 5656 5657 if (queue_index >= channels->num) 5658 goto out; 5659 5660 ch = channels->c[queue_index]; 5661 pdev = ch->pdev; 5662out: 5663 mutex_unlock(&priv->state_lock); 5664 5665 return pdev; 5666} 5667 5668static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { 5669 .ndo_queue_mem_size = sizeof(struct mlx5_qmgmt_data), 5670 .ndo_queue_mem_alloc = mlx5e_queue_mem_alloc, 5671 .ndo_queue_mem_free = mlx5e_queue_mem_free, 5672 .ndo_queue_start = mlx5e_queue_start, 5673 .ndo_queue_stop = mlx5e_queue_stop, 5674 .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, 5675}; 5676 5677static void mlx5e_build_nic_netdev(struct net_device *netdev) 5678{ 5679 struct mlx5e_priv *priv = netdev_priv(netdev); 5680 struct mlx5_core_dev *mdev = priv->mdev; 5681 bool fcs_supported; 5682 bool fcs_enabled; 5683 5684 SET_NETDEV_DEV(netdev, mdev->device); 5685 5686 netdev->netdev_ops = &mlx5e_netdev_ops; 5687 netdev->queue_mgmt_ops = &mlx5e_queue_mgmt_ops; 5688 netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; 5689 netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; 5690 netdev->request_ops_lock = true; 5691 netdev_lockdep_set_classes(netdev); 5692 5693 mlx5e_dcbnl_build_netdev(netdev); 5694 5695 netdev->watchdog_timeo = 15 * HZ; 5696 5697 netdev->stat_ops = &mlx5e_stat_ops; 5698 netdev->ethtool_ops = &mlx5e_ethtool_ops; 5699 5700 netdev->vlan_features |= NETIF_F_SG; 5701 netdev->vlan_features |= NETIF_F_HW_CSUM; 5702 netdev->vlan_features |= NETIF_F_HW_MACSEC; 5703 netdev->vlan_features |= NETIF_F_GRO; 5704 netdev->vlan_features |= NETIF_F_TSO; 5705 netdev->vlan_features |= NETIF_F_TSO6; 5706 netdev->vlan_features |= NETIF_F_RXCSUM; 5707 netdev->vlan_features |= NETIF_F_RXHASH; 5708 netdev->vlan_features |= NETIF_F_GSO_PARTIAL; 5709 5710 netdev->mpls_features |= NETIF_F_SG; 5711 netdev->mpls_features |= NETIF_F_HW_CSUM; 5712 netdev->mpls_features |= NETIF_F_TSO; 5713 netdev->mpls_features |= NETIF_F_TSO6; 5714 5715 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 5716 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 5717 5718 /* Tunneled LRO is not supported in the driver, and the same RQs are 5719 * shared between inner and outer TIRs, so the driver can't disable LRO 5720 * for inner TIRs while having it enabled for outer TIRs. Due to this, 5721 * block LRO altogether if the firmware declares tunneled LRO support. 5722 */ 5723 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 5724 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && 5725 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && 5726 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5727 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5728 netdev->vlan_features |= NETIF_F_LRO; 5729 5730 if (mlx5e_hw_gro_supported(mdev) && 5731 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5732 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5733 netdev->vlan_features |= NETIF_F_GRO_HW; 5734 5735 netdev->hw_features = netdev->vlan_features; 5736 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 5737 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5738 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5739 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5740 5741 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5742 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 5743 netdev->hw_enc_features |= NETIF_F_TSO; 5744 netdev->hw_enc_features |= NETIF_F_TSO6; 5745 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 5746 } 5747 5748 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 5749 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 5750 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5751 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5752 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5753 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 5754 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 5755 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5756 } 5757 5758 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { 5759 netdev->hw_features |= NETIF_F_GSO_GRE | 5760 NETIF_F_GSO_GRE_CSUM; 5761 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 5762 NETIF_F_GSO_GRE_CSUM; 5763 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 5764 NETIF_F_GSO_GRE_CSUM; 5765 } 5766 5767 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { 5768 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 5769 NETIF_F_GSO_IPXIP6; 5770 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 5771 NETIF_F_GSO_IPXIP6; 5772 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 5773 NETIF_F_GSO_IPXIP6; 5774 } 5775 5776 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 5777 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 5778 5779 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 5780 5781 if (fcs_supported) 5782 netdev->hw_features |= NETIF_F_RXALL; 5783 5784 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 5785 netdev->hw_features |= NETIF_F_RXFCS; 5786 5787 if (mlx5_qos_is_supported(mdev)) 5788 netdev->hw_features |= NETIF_F_HW_TC; 5789 5790 netdev->features = netdev->hw_features; 5791 5792 /* Defaults */ 5793 if (fcs_enabled) 5794 netdev->features &= ~NETIF_F_RXALL; 5795 netdev->features &= ~NETIF_F_LRO; 5796 netdev->features &= ~NETIF_F_GRO_HW; 5797 netdev->features &= ~NETIF_F_RXFCS; 5798 5799#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 5800 if (FT_CAP(flow_modify_en) && 5801 FT_CAP(modify_root) && 5802 FT_CAP(identified_miss_table_mode) && 5803 FT_CAP(flow_table_modify)) { 5804#if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 5805 netdev->hw_features |= NETIF_F_HW_TC; 5806#endif 5807#if IS_ENABLED(CONFIG_MLX5_EN_ARFS) 5808 netdev->hw_features |= NETIF_F_NTUPLE; 5809#elif IS_ENABLED(CONFIG_MLX5_EN_RXNFC) 5810 netdev->features |= NETIF_F_NTUPLE; 5811#endif 5812 } 5813 5814 netdev->features |= NETIF_F_HIGHDMA; 5815 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5816 5817 netdev->priv_flags |= IFF_UNICAST_FLT; 5818 5819 netdev->netmem_tx = true; 5820 5821 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 5822 mlx5e_set_xdp_feature(priv); 5823 mlx5e_set_netdev_dev_addr(netdev); 5824 mlx5e_macsec_build_netdev(priv); 5825 mlx5e_ipsec_build_netdev(priv); 5826 mlx5e_ktls_build_netdev(priv); 5827} 5828 5829void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5830{ 5831 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5832 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5833 struct mlx5_core_dev *mdev = priv->mdev; 5834 struct mlx5_core_dev *pos; 5835 int err, i; 5836 5837 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5838 5839 mlx5_sd_for_each_dev(i, mdev, pos) { 5840 err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); 5841 if (!err) 5842 priv->q_counter[i] = 5843 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5844 } 5845 5846 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5847 if (!err) 5848 priv->drop_rq_q_counter = 5849 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5850} 5851 5852void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 5853{ 5854 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 5855 struct mlx5_core_dev *pos; 5856 int i; 5857 5858 MLX5_SET(dealloc_q_counter_in, in, opcode, 5859 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 5860 mlx5_sd_for_each_dev(i, priv->mdev, pos) { 5861 if (priv->q_counter[i]) { 5862 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5863 priv->q_counter[i]); 5864 mlx5_cmd_exec_in(pos, dealloc_q_counter, in); 5865 } 5866 } 5867 5868 if (priv->drop_rq_q_counter) { 5869 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 5870 priv->drop_rq_q_counter); 5871 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 5872 } 5873} 5874 5875static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 5876 struct net_device *netdev) 5877{ 5878 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED; 5879 struct mlx5e_priv *priv = netdev_priv(netdev); 5880 struct mlx5e_flow_steering *fs; 5881 int err; 5882 5883 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); 5884 mlx5e_vxlan_set_netdev_info(priv); 5885 5886 mlx5e_timestamp_init(priv); 5887 5888 priv->dfs_root = debugfs_create_dir("nic", 5889 mlx5_debugfs_get_dev_root(mdev)); 5890 5891 fs = mlx5e_fs_init(priv->profile, mdev, 5892 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 5893 priv->dfs_root); 5894 if (!fs) { 5895 err = -ENOMEM; 5896 mlx5_core_err(mdev, "FS initialization failed, %d\n", err); 5897 debugfs_remove_recursive(priv->dfs_root); 5898 return err; 5899 } 5900 priv->fs = fs; 5901 5902 err = mlx5e_psp_init(priv); 5903 if (err) 5904 mlx5_core_err(mdev, "PSP initialization failed, %d\n", err); 5905 5906 err = mlx5e_ktls_init(priv); 5907 if (err) 5908 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 5909 5910 mlx5e_health_create_reporters(priv); 5911 5912 /* If netdev is already registered (e.g. move from uplink to nic profile), 5913 * RTNL lock must be held before triggering netdev notifiers. 5914 */ 5915 if (take_rtnl) 5916 rtnl_lock(); 5917 5918 mlx5e_psp_register(priv); 5919 /* update XDP supported features */ 5920 mlx5e_set_xdp_feature(priv); 5921 5922 if (take_rtnl) 5923 rtnl_unlock(); 5924 5925 return 0; 5926} 5927 5928static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 5929{ 5930 mlx5e_health_destroy_reporters(priv); 5931 mlx5e_psp_unregister(priv); 5932 mlx5e_ktls_cleanup(priv); 5933 mlx5e_psp_cleanup(priv); 5934 mlx5e_fs_cleanup(priv->fs); 5935 debugfs_remove_recursive(priv->dfs_root); 5936 priv->fs = NULL; 5937} 5938 5939static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 5940{ 5941 struct mlx5_core_dev *mdev = priv->mdev; 5942 enum mlx5e_rx_res_features features; 5943 int err; 5944 5945 mlx5e_create_q_counters(priv); 5946 5947 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 5948 if (err) { 5949 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 5950 goto err_destroy_q_counters; 5951 } 5952 5953 features = MLX5E_RX_RES_FEATURE_PTP; 5954 if (mlx5_tunnel_inner_ft_supported(mdev)) 5955 features |= MLX5E_RX_RES_FEATURE_INNER_FT; 5956 if (mlx5_get_sd(priv->mdev)) 5957 features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; 5958 5959 priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, 5960 &priv->channels.params.packet_merge, 5961 priv->channels.params.num_channels); 5962 if (IS_ERR(priv->rx_res)) { 5963 err = PTR_ERR(priv->rx_res); 5964 priv->rx_res = NULL; 5965 mlx5_core_err(mdev, "create rx resources failed, %d\n", err); 5966 goto err_close_drop_rq; 5967 } 5968 5969 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, 5970 priv->netdev); 5971 if (err) { 5972 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 5973 goto err_destroy_rx_res; 5974 } 5975 5976 err = mlx5e_tc_nic_init(priv); 5977 if (err) 5978 goto err_destroy_flow_steering; 5979 5980 err = mlx5e_accel_init_rx(priv); 5981 if (err) 5982 goto err_tc_nic_cleanup; 5983 5984#ifdef CONFIG_MLX5_EN_ARFS 5985 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 5986#endif 5987 5988 return 0; 5989 5990err_tc_nic_cleanup: 5991 mlx5e_tc_nic_cleanup(priv); 5992err_destroy_flow_steering: 5993 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 5994 priv->profile); 5995err_destroy_rx_res: 5996 mlx5e_rx_res_destroy(priv->rx_res); 5997 priv->rx_res = NULL; 5998err_close_drop_rq: 5999 mlx5e_close_drop_rq(&priv->drop_rq); 6000err_destroy_q_counters: 6001 mlx5e_destroy_q_counters(priv); 6002 return err; 6003} 6004 6005static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 6006{ 6007 mlx5e_accel_cleanup_rx(priv); 6008 mlx5e_tc_nic_cleanup(priv); 6009 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 6010 priv->profile); 6011 mlx5e_rx_res_destroy(priv->rx_res); 6012 priv->rx_res = NULL; 6013 mlx5e_close_drop_rq(&priv->drop_rq); 6014 mlx5e_destroy_q_counters(priv); 6015} 6016 6017static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) 6018{ 6019 struct mlx5e_params *params; 6020 struct mlx5e_mqprio_rl *rl; 6021 6022 params = &priv->channels.params; 6023 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) 6024 return; 6025 6026 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, 6027 params->mqprio.channel.max_rate); 6028 if (IS_ERR(rl)) 6029 rl = NULL; 6030 priv->mqprio_rl = rl; 6031 mlx5e_mqprio_rl_update_params(params, rl); 6032} 6033 6034static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 6035{ 6036 int err; 6037 6038 err = mlx5e_accel_init_tx(priv); 6039 if (err) 6040 return err; 6041 6042 mlx5e_set_mqprio_rl(priv); 6043 mlx5e_dcbnl_initialize(priv); 6044 return 0; 6045} 6046 6047static void mlx5e_nic_enable(struct mlx5e_priv *priv) 6048{ 6049 struct net_device *netdev = priv->netdev; 6050 struct mlx5_core_dev *mdev = priv->mdev; 6051 int err; 6052 6053 mlx5e_fs_init_l2_addr(priv->fs, netdev); 6054 mlx5e_ipsec_init(priv); 6055 6056 err = mlx5e_macsec_init(priv); 6057 if (err) 6058 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); 6059 6060 /* Marking the link as currently not needed by the Driver */ 6061 if (!netif_running(netdev)) 6062 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 6063 6064 mlx5e_set_netdev_mtu_boundaries(priv); 6065 mlx5e_set_dev_port_mtu(priv); 6066 6067 mlx5_lag_add_netdev(mdev, netdev); 6068 6069 mlx5e_enable_async_events(priv); 6070 mlx5e_enable_blocking_events(priv); 6071 if (mlx5e_monitor_counter_supported(priv)) 6072 mlx5e_monitor_counter_init(priv); 6073 6074 mlx5e_pcie_cong_event_init(priv); 6075 mlx5e_hv_vhca_stats_create(priv); 6076 if (netdev->reg_state != NETREG_REGISTERED) 6077 return; 6078 mlx5e_dcbnl_init_app(priv); 6079 6080 mlx5e_nic_set_rx_mode(priv); 6081 6082 rtnl_lock(); 6083 netdev_lock(netdev); 6084 if (netif_running(netdev)) 6085 mlx5e_open(netdev); 6086 udp_tunnel_nic_reset_ntf(priv->netdev); 6087 netdev_unlock(netdev); 6088 netif_device_attach(netdev); 6089 rtnl_unlock(); 6090} 6091 6092static void mlx5e_nic_disable(struct mlx5e_priv *priv) 6093{ 6094 struct mlx5_core_dev *mdev = priv->mdev; 6095 6096 if (priv->netdev->reg_state == NETREG_REGISTERED) 6097 mlx5e_dcbnl_delete_app(priv); 6098 6099 rtnl_lock(); 6100 netdev_lock(priv->netdev); 6101 if (netif_running(priv->netdev)) 6102 mlx5e_close(priv->netdev); 6103 netif_device_detach(priv->netdev); 6104 if (priv->en_trap) { 6105 mlx5e_deactivate_trap(priv); 6106 mlx5e_close_trap(priv->en_trap); 6107 priv->en_trap = NULL; 6108 } 6109 netdev_unlock(priv->netdev); 6110 rtnl_unlock(); 6111 6112 mlx5e_nic_set_rx_mode(priv); 6113 6114 mlx5e_pcie_cong_event_cleanup(priv); 6115 mlx5e_hv_vhca_stats_destroy(priv); 6116 if (mlx5e_monitor_counter_supported(priv)) 6117 mlx5e_monitor_counter_cleanup(priv); 6118 6119 mlx5e_ipsec_disable_events(priv); 6120 mlx5e_disable_blocking_events(priv); 6121 mlx5e_disable_async_events(priv); 6122 mlx5_lag_remove_netdev(mdev, priv->netdev); 6123 mlx5_vxlan_reset_to_default(mdev->vxlan); 6124 mlx5e_macsec_cleanup(priv); 6125 mlx5e_ipsec_cleanup(priv); 6126} 6127 6128static int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 6129{ 6130 return mlx5e_refresh_tirs(priv->mdev, false, false); 6131} 6132 6133static const struct mlx5e_profile mlx5e_nic_profile = { 6134 .init = mlx5e_nic_init, 6135 .cleanup = mlx5e_nic_cleanup, 6136 .init_rx = mlx5e_init_nic_rx, 6137 .cleanup_rx = mlx5e_cleanup_nic_rx, 6138 .init_tx = mlx5e_init_nic_tx, 6139 .cleanup_tx = mlx5e_cleanup_nic_tx, 6140 .enable = mlx5e_nic_enable, 6141 .disable = mlx5e_nic_disable, 6142 .update_rx = mlx5e_update_nic_rx, 6143 .update_stats = mlx5e_stats_update_ndo_stats, 6144 .update_carrier = mlx5e_update_carrier, 6145 .rx_handlers = &mlx5e_rx_handlers_nic, 6146 .max_tc = MLX5_MAX_NUM_TC, 6147 .stats_grps = mlx5e_nic_stats_grps, 6148 .stats_grps_num = mlx5e_nic_stats_grps_num, 6149 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | 6150 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | 6151 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | 6152 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | 6153 BIT(MLX5E_PROFILE_FEATURE_FS_TC), 6154}; 6155 6156static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, 6157 const struct mlx5e_profile *profile) 6158{ 6159 int nch; 6160 6161 nch = mlx5e_get_max_num_channels(mdev); 6162 6163 if (profile->max_nch_limit) 6164 nch = min_t(int, nch, profile->max_nch_limit(mdev)); 6165 return nch; 6166} 6167 6168static unsigned int 6169mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, 6170 const struct mlx5e_profile *profile) 6171 6172{ 6173 unsigned int max_nch, tmp; 6174 6175 /* core resources */ 6176 max_nch = mlx5e_profile_max_num_channels(mdev, profile); 6177 6178 /* netdev rx queues */ 6179 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); 6180 6181 /* netdev tx queues */ 6182 tmp = netdev->num_tx_queues; 6183 if (mlx5_qos_is_supported(mdev)) 6184 tmp -= mlx5e_qos_max_leaf_nodes(mdev); 6185 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) 6186 tmp -= profile->max_tc; 6187 tmp = tmp / profile->max_tc; 6188 max_nch = min_t(unsigned int, max_nch, tmp); 6189 6190 return max_nch; 6191} 6192 6193int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) 6194{ 6195 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering) 6196 * and 1 set of direct TIRS 6197 */ 6198 return 2 * MLX5E_NUM_INDIR_TIRS 6199 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); 6200} 6201 6202void mlx5e_set_rx_mode_work(struct work_struct *work) 6203{ 6204 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 6205 set_rx_mode_work); 6206 6207 return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev); 6208} 6209 6210/* mlx5e generic netdev management API (move to en_common.c) */ 6211int mlx5e_priv_init(struct mlx5e_priv *priv, 6212 const struct mlx5e_profile *profile, 6213 struct net_device *netdev, 6214 struct mlx5_core_dev *mdev) 6215{ 6216 int nch, num_txqs, node; 6217 int err; 6218 6219 num_txqs = netdev->num_tx_queues; 6220 nch = mlx5e_calc_max_nch(mdev, netdev, profile); 6221 node = dev_to_node(mlx5_core_dma_dev(mdev)); 6222 6223 /* priv init */ 6224 priv->mdev = mdev; 6225 priv->netdev = netdev; 6226 priv->max_nch = nch; 6227 priv->max_opened_tc = 1; 6228 6229 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 6230 return -ENOMEM; 6231 6232 mutex_init(&priv->state_lock); 6233 6234 err = mlx5e_selq_init(&priv->selq, &priv->state_lock); 6235 if (err) 6236 goto err_free_cpumask; 6237 6238 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 6239 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 6240 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 6241 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 6242 6243 priv->wq = create_singlethread_workqueue("mlx5e"); 6244 if (!priv->wq) 6245 goto err_free_selq; 6246 6247 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node); 6248 if (!priv->txq2sq) 6249 goto err_destroy_workqueue; 6250 6251 priv->txq2sq_stats = kcalloc_node(num_txqs, sizeof(*priv->txq2sq_stats), GFP_KERNEL, node); 6252 if (!priv->txq2sq_stats) 6253 goto err_free_txq2sq; 6254 6255 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node); 6256 if (!priv->tx_rates) 6257 goto err_free_txq2sq_stats; 6258 6259 priv->channel_stats = 6260 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node); 6261 if (!priv->channel_stats) 6262 goto err_free_tx_rates; 6263 6264 priv->fec_ranges = kcalloc(ETHTOOL_FEC_HIST_MAX, 6265 sizeof(*priv->fec_ranges), GFP_KERNEL); 6266 if (!priv->fec_ranges) 6267 goto err_free_channel_stats; 6268 6269 return 0; 6270 6271err_free_channel_stats: 6272 kfree(priv->channel_stats); 6273err_free_tx_rates: 6274 kfree(priv->tx_rates); 6275err_free_txq2sq_stats: 6276 kfree(priv->txq2sq_stats); 6277err_free_txq2sq: 6278 kfree(priv->txq2sq); 6279err_destroy_workqueue: 6280 destroy_workqueue(priv->wq); 6281err_free_selq: 6282 mlx5e_selq_cleanup(&priv->selq); 6283err_free_cpumask: 6284 free_cpumask_var(priv->scratchpad.cpumask); 6285 return -ENOMEM; 6286} 6287 6288void mlx5e_priv_cleanup(struct mlx5e_priv *priv) 6289{ 6290 bool destroying = test_bit(MLX5E_STATE_DESTROYING, &priv->state); 6291 int i; 6292 6293 /* bail if change profile failed and also rollback failed */ 6294 if (!priv->mdev) 6295 return; 6296 6297 kfree(priv->fec_ranges); 6298 for (i = 0; i < priv->stats_nch; i++) 6299 kvfree(priv->channel_stats[i]); 6300 kfree(priv->channel_stats); 6301 kfree(priv->tx_rates); 6302 kfree(priv->txq2sq_stats); 6303 kfree(priv->txq2sq); 6304 destroy_workqueue(priv->wq); 6305 mlx5e_selq_cleanup(&priv->selq); 6306 free_cpumask_var(priv->scratchpad.cpumask); 6307 6308 for (i = 0; i < priv->htb_max_qos_sqs; i++) 6309 kfree(priv->htb_qos_sq_stats[i]); 6310 kvfree(priv->htb_qos_sq_stats); 6311 6312 if (priv->mqprio_rl) { 6313 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 6314 mlx5e_mqprio_rl_free(priv->mqprio_rl); 6315 } 6316 6317 memset(priv, 0, sizeof(*priv)); 6318 if (destroying) /* restore destroying bit, to allow unload */ 6319 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6320} 6321 6322static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, 6323 const struct mlx5e_profile *profile) 6324{ 6325 unsigned int nch, ptp_txqs, qos_txqs; 6326 6327 nch = mlx5e_profile_max_num_channels(mdev, profile); 6328 6329 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) && 6330 mlx5e_profile_feature_cap(profile, PTP_TX) ? 6331 profile->max_tc : 0; 6332 6333 qos_txqs = mlx5_qos_is_supported(mdev) && 6334 mlx5e_profile_feature_cap(profile, QOS_HTB) ? 6335 mlx5e_qos_max_leaf_nodes(mdev) : 0; 6336 6337 return nch * profile->max_tc + ptp_txqs + qos_txqs; 6338} 6339 6340static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, 6341 const struct mlx5e_profile *profile) 6342{ 6343 return mlx5e_profile_max_num_channels(mdev, profile); 6344} 6345 6346struct net_device * 6347mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) 6348{ 6349 struct net_device *netdev; 6350 unsigned int txqs, rxqs; 6351 int err; 6352 6353 txqs = mlx5e_get_max_num_txqs(mdev, profile); 6354 rxqs = mlx5e_get_max_num_rxqs(mdev, profile); 6355 6356 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs); 6357 if (!netdev) { 6358 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 6359 return NULL; 6360 } 6361 6362 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev); 6363 if (err) { 6364 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6365 goto err_free_netdev; 6366 } 6367 6368 netif_carrier_off(netdev); 6369 netif_tx_disable(netdev); 6370 dev_net_set(netdev, mlx5_core_net(mdev)); 6371 6372 return netdev; 6373 6374err_free_netdev: 6375 free_netdev(netdev); 6376 6377 return NULL; 6378} 6379 6380static void mlx5e_update_features(struct net_device *netdev) 6381{ 6382 if (netdev->reg_state != NETREG_REGISTERED) 6383 return; /* features will be updated on netdev registration */ 6384 6385 rtnl_lock(); 6386 netdev_lock(netdev); 6387 netdev_update_features(netdev); 6388 netdev_unlock(netdev); 6389 rtnl_unlock(); 6390} 6391 6392static void mlx5e_reset_channels(struct net_device *netdev) 6393{ 6394 netdev_reset_tc(netdev); 6395} 6396 6397int mlx5e_attach_netdev(struct mlx5e_priv *priv) 6398{ 6399 const bool need_lock = priv->netdev->reg_state == NETREG_REGISTERED; 6400 const struct mlx5e_profile *profile = priv->profile; 6401 int max_nch; 6402 int err; 6403 6404 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 6405 if (priv->fs) 6406 mlx5e_fs_set_state_destroy(priv->fs, 6407 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6408 6409 /* Validate the max_wqe_size_sq capability. */ 6410 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { 6411 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n", 6412 mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS); 6413 return -EIO; 6414 } 6415 6416 /* max number of channels may have changed */ 6417 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); 6418 if (priv->channels.params.num_channels > max_nch) { 6419 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 6420 /* Reducing the number of channels - RXFH has to be reset, and 6421 * mlx5e_num_channels_changed below will build the RQT. 6422 */ 6423 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED; 6424 priv->channels.params.num_channels = max_nch; 6425 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { 6426 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); 6427 mlx5e_params_mqprio_reset(&priv->channels.params); 6428 } 6429 } 6430 if (max_nch != priv->max_nch) { 6431 mlx5_core_warn(priv->mdev, 6432 "MLX5E: Updating max number of channels from %u to %u\n", 6433 priv->max_nch, max_nch); 6434 priv->max_nch = max_nch; 6435 } 6436 6437 /* 1. Set the real number of queues in the kernel the first time. 6438 * 2. Set our default XPS cpumask. 6439 * 3. Build the RQT. 6440 * 6441 * Locking is required by netif_set_real_num_*_queues in case the 6442 * netdev has been registered by this point (if this function was called 6443 * in the reload or resume flow). 6444 */ 6445 if (need_lock) { 6446 rtnl_lock(); 6447 netdev_lock(priv->netdev); 6448 } 6449 err = mlx5e_num_channels_changed(priv); 6450 if (need_lock) { 6451 netdev_unlock(priv->netdev); 6452 rtnl_unlock(); 6453 } 6454 if (err) 6455 goto out; 6456 6457 err = profile->init_tx(priv); 6458 if (err) 6459 goto out; 6460 6461 err = profile->init_rx(priv); 6462 if (err) 6463 goto err_cleanup_tx; 6464 6465 if (profile->enable) 6466 profile->enable(priv); 6467 6468 mlx5e_update_features(priv->netdev); 6469 6470 return 0; 6471 6472err_cleanup_tx: 6473 profile->cleanup_tx(priv); 6474 6475out: 6476 mlx5e_reset_channels(priv->netdev); 6477 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6478 if (priv->fs) 6479 mlx5e_fs_set_state_destroy(priv->fs, 6480 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6481 cancel_work_sync(&priv->update_stats_work); 6482 return err; 6483} 6484 6485void mlx5e_detach_netdev(struct mlx5e_priv *priv) 6486{ 6487 const struct mlx5e_profile *profile = priv->profile; 6488 6489 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6490 if (priv->fs) 6491 mlx5e_fs_set_state_destroy(priv->fs, 6492 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6493 6494 if (profile->disable) 6495 profile->disable(priv); 6496 flush_workqueue(priv->wq); 6497 6498 profile->cleanup_rx(priv); 6499 profile->cleanup_tx(priv); 6500 mlx5e_reset_channels(priv->netdev); 6501 cancel_work_sync(&priv->update_stats_work); 6502} 6503 6504static int 6505mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6506 const struct mlx5e_profile *new_profile, void *new_ppriv) 6507{ 6508 struct mlx5e_priv *priv = netdev_priv(netdev); 6509 int err; 6510 6511 err = mlx5e_priv_init(priv, new_profile, netdev, mdev); 6512 if (err) { 6513 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6514 return err; 6515 } 6516 netif_carrier_off(netdev); 6517 priv->profile = new_profile; 6518 priv->ppriv = new_ppriv; 6519 err = new_profile->init(priv->mdev, priv->netdev); 6520 if (err) 6521 goto priv_cleanup; 6522 6523 return 0; 6524 6525priv_cleanup: 6526 mlx5e_priv_cleanup(priv); 6527 return err; 6528} 6529 6530static int 6531mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6532 const struct mlx5e_profile *new_profile, void *new_ppriv) 6533{ 6534 struct mlx5e_priv *priv = netdev_priv(netdev); 6535 int err; 6536 6537 err = mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6538 if (err) 6539 return err; 6540 6541 err = mlx5e_attach_netdev(priv); 6542 if (err) 6543 goto profile_cleanup; 6544 return err; 6545 6546profile_cleanup: 6547 new_profile->cleanup(priv); 6548 mlx5e_priv_cleanup(priv); 6549 return err; 6550} 6551 6552int mlx5e_netdev_change_profile(struct net_device *netdev, 6553 struct mlx5_core_dev *mdev, 6554 const struct mlx5e_profile *new_profile, 6555 void *new_ppriv) 6556{ 6557 struct mlx5e_priv *priv = netdev_priv(netdev); 6558 const struct mlx5e_profile *orig_profile; 6559 int err, rollback_err; 6560 void *orig_ppriv; 6561 6562 orig_profile = priv->profile; 6563 orig_ppriv = priv->ppriv; 6564 6565 /* NULL could happen if previous change_profile failed to rollback */ 6566 if (priv->profile) { 6567 WARN_ON_ONCE(priv->mdev != mdev); 6568 /* cleanup old profile */ 6569 mlx5e_detach_netdev(priv); 6570 priv->profile->cleanup(priv); 6571 mlx5e_priv_cleanup(priv); 6572 } 6573 /* priv members are not valid from this point ... */ 6574 6575 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 6576 mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6577 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6578 return -EIO; 6579 } 6580 6581 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv); 6582 if (err) { /* roll back to original profile */ 6583 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err); 6584 goto rollback; 6585 } 6586 6587 return 0; 6588 6589rollback: 6590 if (!orig_profile) { 6591 netdev_warn(netdev, "no original profile to rollback to\n"); 6592 priv->profile = NULL; 6593 return err; 6594 } 6595 6596 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); 6597 if (rollback_err) { 6598 netdev_err(netdev, "failed to rollback to orig profile, %d\n", 6599 rollback_err); 6600 priv->profile = NULL; 6601 } 6602 return err; 6603} 6604 6605void mlx5e_netdev_attach_nic_profile(struct net_device *netdev, 6606 struct mlx5_core_dev *mdev) 6607{ 6608 mlx5e_netdev_change_profile(netdev, mdev, &mlx5e_nic_profile, NULL); 6609} 6610 6611void mlx5e_destroy_netdev(struct net_device *netdev) 6612{ 6613 struct mlx5e_priv *priv = netdev_priv(netdev); 6614 6615 if (priv->profile) 6616 mlx5e_priv_cleanup(priv); 6617 free_netdev(netdev); 6618} 6619 6620static int _mlx5e_resume(struct auxiliary_device *adev) 6621{ 6622 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6623 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6624 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6625 struct net_device *netdev = mlx5e_dev->netdev; 6626 struct mlx5_core_dev *mdev = edev->mdev; 6627 struct mlx5_core_dev *pos, *to; 6628 int err, i; 6629 6630 if (netif_device_present(netdev)) 6631 return 0; 6632 6633 mlx5_sd_for_each_dev(i, mdev, pos) { 6634 err = mlx5e_create_mdev_resources(pos, true); 6635 if (err) 6636 goto err_destroy_mdev_res; 6637 } 6638 6639 err = mlx5e_attach_netdev(priv); 6640 if (err) 6641 goto err_destroy_mdev_res; 6642 6643 return 0; 6644 6645err_destroy_mdev_res: 6646 to = pos; 6647 mlx5_sd_for_each_dev_to(i, mdev, to, pos) 6648 mlx5e_destroy_mdev_resources(pos); 6649 return err; 6650} 6651 6652static int mlx5e_resume(struct auxiliary_device *adev) 6653{ 6654 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6655 struct mlx5_core_dev *mdev = edev->mdev; 6656 struct auxiliary_device *actual_adev; 6657 int err; 6658 6659 err = mlx5_sd_init(mdev); 6660 if (err) 6661 return err; 6662 6663 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6664 if (actual_adev) 6665 return _mlx5e_resume(actual_adev); 6666 return 0; 6667} 6668 6669static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg) 6670{ 6671 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6672 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6673 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6674 struct net_device *netdev = mlx5e_dev->netdev; 6675 struct mlx5_core_dev *mdev = edev->mdev; 6676 struct mlx5_core_dev *pos; 6677 int i; 6678 6679 if (!pre_netdev_reg && !netif_device_present(netdev)) { 6680 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 6681 mlx5_sd_for_each_dev(i, mdev, pos) 6682 mlx5e_destroy_mdev_resources(pos); 6683 return -ENODEV; 6684 } 6685 6686 mlx5e_detach_netdev(priv); 6687 mlx5_sd_for_each_dev(i, mdev, pos) 6688 mlx5e_destroy_mdev_resources(pos); 6689 6690 return 0; 6691} 6692 6693static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) 6694{ 6695 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6696 struct mlx5_core_dev *mdev = edev->mdev; 6697 struct auxiliary_device *actual_adev; 6698 int err = 0; 6699 6700 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6701 if (actual_adev) 6702 err = _mlx5e_suspend(actual_adev, false); 6703 6704 mlx5_sd_cleanup(mdev); 6705 return err; 6706} 6707 6708static int _mlx5e_probe(struct auxiliary_device *adev) 6709{ 6710 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6711 const struct mlx5e_profile *profile = &mlx5e_nic_profile; 6712 struct mlx5_core_dev *mdev = edev->mdev; 6713 struct mlx5e_dev *mlx5e_dev; 6714 struct net_device *netdev; 6715 struct mlx5e_priv *priv; 6716 int err; 6717 6718 mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); 6719 if (IS_ERR(mlx5e_dev)) 6720 return PTR_ERR(mlx5e_dev); 6721 auxiliary_set_drvdata(adev, mlx5e_dev); 6722 6723 err = mlx5e_devlink_port_register(mlx5e_dev, mdev); 6724 if (err) { 6725 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 6726 goto err_devlink_unregister; 6727 } 6728 6729 netdev = mlx5e_create_netdev(mdev, profile); 6730 if (!netdev) { 6731 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 6732 err = -ENOMEM; 6733 goto err_devlink_port_unregister; 6734 } 6735 SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port); 6736 mlx5e_dev->netdev = netdev; 6737 6738 mlx5e_build_nic_netdev(netdev); 6739 6740 priv = netdev_priv(netdev); 6741 6742 priv->profile = profile; 6743 priv->ppriv = NULL; 6744 6745 err = profile->init(mdev, netdev); 6746 if (err) { 6747 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err); 6748 goto err_destroy_netdev; 6749 } 6750 6751 err = _mlx5e_resume(adev); 6752 if (err) { 6753 mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); 6754 goto err_profile_cleanup; 6755 } 6756 6757 err = register_netdev(netdev); 6758 if (err) { 6759 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 6760 goto err_resume; 6761 } 6762 6763 mlx5e_dcbnl_init_app(priv); 6764 mlx5_core_uplink_netdev_set(mdev, netdev); 6765 mlx5e_params_print_info(mdev, &priv->channels.params); 6766 return 0; 6767 6768err_resume: 6769 _mlx5e_suspend(adev, true); 6770err_profile_cleanup: 6771 profile->cleanup(priv); 6772err_destroy_netdev: 6773 mlx5e_destroy_netdev(netdev); 6774err_devlink_port_unregister: 6775 mlx5e_devlink_port_unregister(mlx5e_dev); 6776err_devlink_unregister: 6777 mlx5e_destroy_devlink(mlx5e_dev); 6778 return err; 6779} 6780 6781static int mlx5e_probe(struct auxiliary_device *adev, 6782 const struct auxiliary_device_id *id) 6783{ 6784 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6785 struct mlx5_core_dev *mdev = edev->mdev; 6786 struct auxiliary_device *actual_adev; 6787 int err; 6788 6789 err = mlx5_sd_init(mdev); 6790 if (err) 6791 return err; 6792 6793 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6794 if (actual_adev) 6795 return _mlx5e_probe(actual_adev); 6796 return 0; 6797} 6798 6799static void _mlx5e_remove(struct auxiliary_device *adev) 6800{ 6801 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6802 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6803 struct net_device *netdev = mlx5e_dev->netdev; 6804 struct mlx5e_priv *priv = netdev_priv(netdev); 6805 struct mlx5_core_dev *mdev = edev->mdev; 6806 6807 mlx5_eswitch_safe_aux_devs_remove(mdev); 6808 mlx5_core_uplink_netdev_set(mdev, NULL); 6809 6810 if (priv->profile) 6811 mlx5e_dcbnl_delete_app(priv); 6812 /* When unload driver, the netdev is in registered state 6813 * if it's from legacy mode. If from switchdev mode, it 6814 * is already unregistered before changing to NIC profile. 6815 */ 6816 if (netdev->reg_state == NETREG_REGISTERED) { 6817 unregister_netdev(netdev); 6818 _mlx5e_suspend(adev, false); 6819 } else { 6820 struct mlx5_core_dev *pos; 6821 int i; 6822 6823 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 6824 mlx5_sd_for_each_dev(i, mdev, pos) 6825 mlx5e_destroy_mdev_resources(pos); 6826 else 6827 _mlx5e_suspend(adev, true); 6828 } 6829 /* Avoid cleanup if profile rollback failed. */ 6830 if (priv->profile) 6831 priv->profile->cleanup(priv); 6832 mlx5e_destroy_netdev(netdev); 6833 mlx5e_devlink_port_unregister(mlx5e_dev); 6834 mlx5e_destroy_devlink(mlx5e_dev); 6835} 6836 6837static void mlx5e_remove(struct auxiliary_device *adev) 6838{ 6839 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6840 struct mlx5_core_dev *mdev = edev->mdev; 6841 struct auxiliary_device *actual_adev; 6842 6843 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6844 if (actual_adev) 6845 _mlx5e_remove(actual_adev); 6846 6847 mlx5_sd_cleanup(mdev); 6848} 6849 6850static const struct auxiliary_device_id mlx5e_id_table[] = { 6851 { .name = MLX5_ADEV_NAME ".eth", }, 6852 {}, 6853}; 6854 6855MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table); 6856 6857static struct auxiliary_driver mlx5e_driver = { 6858 .name = "eth", 6859 .probe = mlx5e_probe, 6860 .remove = mlx5e_remove, 6861 .suspend = mlx5e_suspend, 6862 .resume = mlx5e_resume, 6863 .id_table = mlx5e_id_table, 6864}; 6865 6866int mlx5e_init(void) 6867{ 6868 int ret; 6869 6870 mlx5e_build_ptys2ethtool_map(); 6871 ret = auxiliary_driver_register(&mlx5e_driver); 6872 if (ret) 6873 return ret; 6874 6875 ret = mlx5e_rep_init(); 6876 if (ret) 6877 auxiliary_driver_unregister(&mlx5e_driver); 6878 return ret; 6879} 6880 6881void mlx5e_cleanup(void) 6882{ 6883 mlx5e_rep_cleanup(); 6884 auxiliary_driver_unregister(&mlx5e_driver); 6885}
~/
Read-only file preview.