FILE
hns_roce_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/infiniband/hw/hns/hns_roce_main.c
1/* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33#include <linux/acpi.h> 34#include <linux/module.h> 35#include <rdma/ib_addr.h> 36#include <rdma/ib_smi.h> 37#include <rdma/ib_user_verbs.h> 38#include <rdma/ib_cache.h> 39#include "hns_roce_common.h" 40#include "hns_roce_device.h" 41#include "hns_roce_hem.h" 42#include "hns_roce_hw_v2.h" 43#include "hns_roce_bond.h" 44 45static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u32 port, 46 const u8 *addr) 47{ 48 u8 phy_port; 49 u32 i; 50 51 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 52 return 0; 53 54 if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN)) 55 return 0; 56 57 for (i = 0; i < ETH_ALEN; i++) 58 hr_dev->dev_addr[port][i] = addr[i]; 59 60 phy_port = hr_dev->iboe.phy_port[port]; 61 return hr_dev->hw->set_mac(hr_dev, phy_port, addr); 62} 63 64static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) 65{ 66 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); 67 u32 port = attr->port_num - 1; 68 int ret; 69 70 if (port >= hr_dev->caps.num_ports) 71 return -EINVAL; 72 73 ret = hr_dev->hw->set_gid(hr_dev, attr->index, &attr->gid, attr); 74 75 return ret; 76} 77 78static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) 79{ 80 struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); 81 u32 port = attr->port_num - 1; 82 int ret; 83 84 if (port >= hr_dev->caps.num_ports) 85 return -EINVAL; 86 87 ret = hr_dev->hw->set_gid(hr_dev, attr->index, NULL, NULL); 88 89 return ret; 90} 91 92static int hns_roce_get_port_state(struct hns_roce_dev *hr_dev, u32 port_num, 93 enum ib_port_state *state) 94{ 95 struct hns_roce_bond_group *bond_grp; 96 u8 bus_num = get_hr_bus_num(hr_dev); 97 struct net_device *net_dev; 98 99 net_dev = ib_device_get_netdev(&hr_dev->ib_dev, port_num); 100 if (!net_dev) 101 return -ENODEV; 102 103 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { 104 bond_grp = hns_roce_get_bond_grp(net_dev, bus_num); 105 if (bond_grp) { 106 *state = ib_get_curr_port_state(bond_grp->upper_dev); 107 goto out; 108 } 109 } 110 111 *state = ib_get_curr_port_state(net_dev); 112out: 113 dev_put(net_dev); 114 return 0; 115} 116 117static int handle_en_event(struct net_device *netdev, 118 struct hns_roce_dev *hr_dev, 119 u32 port, unsigned long event) 120{ 121 struct ib_device *ibdev = &hr_dev->ib_dev; 122 struct device *dev = hr_dev->dev; 123 enum ib_port_state curr_state; 124 struct ib_event ibevent; 125 int ret = 0; 126 127 if (!netdev) { 128 dev_err(dev, "can't find netdev on port(%u)!\n", port); 129 return -ENODEV; 130 } 131 132 switch (event) { 133 case NETDEV_REGISTER: 134 case NETDEV_CHANGEADDR: 135 ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr); 136 break; 137 case NETDEV_UP: 138 case NETDEV_CHANGE: 139 ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr); 140 if (ret) 141 return ret; 142 fallthrough; 143 case NETDEV_DOWN: 144 if (!netif_is_lag_master(netdev)) 145 break; 146 curr_state = ib_get_curr_port_state(netdev); 147 148 write_lock_irq(&ibdev->cache_lock); 149 if (ibdev->port_data[port].cache.last_port_state == curr_state) { 150 write_unlock_irq(&ibdev->cache_lock); 151 return 0; 152 } 153 ibdev->port_data[port].cache.last_port_state = curr_state; 154 write_unlock_irq(&ibdev->cache_lock); 155 156 ibevent.event = (curr_state == IB_PORT_DOWN) ? 157 IB_EVENT_PORT_ERR : IB_EVENT_PORT_ACTIVE; 158 ibevent.device = ibdev; 159 ibevent.element.port_num = port + 1; 160 ib_dispatch_event(&ibevent); 161 break; 162 default: 163 dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event)); 164 break; 165 } 166 167 return ret; 168} 169 170static int hns_roce_netdev_event(struct notifier_block *self, 171 unsigned long event, void *ptr) 172{ 173 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 174 struct hns_roce_bond_group *bond_grp; 175 struct hns_roce_ib_iboe *iboe = NULL; 176 struct hns_roce_dev *hr_dev = NULL; 177 struct net_device *upper = NULL; 178 int ret; 179 u32 port; 180 181 hr_dev = container_of(self, struct hns_roce_dev, iboe.nb); 182 iboe = &hr_dev->iboe; 183 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { 184 bond_grp = hns_roce_get_bond_grp(get_hr_netdev(hr_dev, 0), 185 get_hr_bus_num(hr_dev)); 186 upper = bond_grp ? bond_grp->upper_dev : NULL; 187 } 188 189 for (port = 0; port < hr_dev->caps.num_ports; port++) { 190 if ((!upper && dev == iboe->netdevs[port]) || 191 (upper && dev == upper)) { 192 ret = handle_en_event(dev, hr_dev, port, event); 193 if (ret) 194 return NOTIFY_DONE; 195 break; 196 } 197 } 198 199 return NOTIFY_DONE; 200} 201 202static int hns_roce_setup_mtu_mac(struct hns_roce_dev *hr_dev) 203{ 204 struct net_device *net_dev; 205 int ret; 206 u8 i; 207 208 for (i = 0; i < hr_dev->caps.num_ports; i++) { 209 net_dev = get_hr_netdev(hr_dev, i); 210 ret = hns_roce_set_mac(hr_dev, i, net_dev->dev_addr); 211 if (ret) 212 return ret; 213 } 214 215 return 0; 216} 217 218static int hns_roce_query_device(struct ib_device *ib_dev, 219 struct ib_device_attr *props, 220 struct ib_udata *uhw) 221{ 222 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); 223 224 memset(props, 0, sizeof(*props)); 225 226 props->fw_ver = hr_dev->caps.fw_ver; 227 props->sys_image_guid = cpu_to_be64(hr_dev->sys_image_guid); 228 props->max_mr_size = (u64)(~(0ULL)); 229 props->page_size_cap = hr_dev->caps.page_size_cap; 230 props->vendor_id = hr_dev->vendor_id; 231 props->vendor_part_id = hr_dev->vendor_part_id; 232 props->hw_ver = hr_dev->hw_rev; 233 props->max_qp = hr_dev->caps.num_qps; 234 props->max_qp_wr = hr_dev->caps.max_wqes; 235 props->device_cap_flags = IB_DEVICE_PORT_ACTIVE_EVENT | 236 IB_DEVICE_RC_RNR_NAK_GEN; 237 props->max_send_sge = hr_dev->caps.max_sq_sg; 238 props->max_recv_sge = hr_dev->caps.max_rq_sg; 239 props->max_sge_rd = hr_dev->caps.max_sq_sg; 240 props->max_cq = hr_dev->caps.num_cqs; 241 props->max_cqe = hr_dev->caps.max_cqes; 242 props->max_mr = hr_dev->caps.num_mtpts; 243 props->max_pd = hr_dev->caps.num_pds; 244 props->max_qp_rd_atom = hr_dev->caps.max_qp_dest_rdma; 245 props->max_qp_init_rd_atom = hr_dev->caps.max_qp_init_rdma; 246 props->atomic_cap = hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_ATOMIC ? 247 IB_ATOMIC_HCA : IB_ATOMIC_NONE; 248 props->max_pkeys = 1; 249 props->local_ca_ack_delay = hr_dev->caps.local_ca_ack_delay; 250 props->max_ah = INT_MAX; 251 props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD; 252 props->cq_caps.max_cq_moderation_count = HNS_ROCE_MAX_CQ_COUNT; 253 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) 254 props->cq_caps.max_cq_moderation_period = HNS_ROCE_MAX_CQ_PERIOD_HIP08; 255 256 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { 257 props->max_srq = hr_dev->caps.num_srqs; 258 props->max_srq_wr = hr_dev->caps.max_srq_wrs; 259 props->max_srq_sge = hr_dev->caps.max_srq_sges; 260 } 261 262 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR && 263 hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { 264 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; 265 props->max_fast_reg_page_list_len = HNS_ROCE_FRMR_MAX_PA; 266 } 267 268 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) 269 props->device_cap_flags |= IB_DEVICE_XRC; 270 271 return 0; 272} 273 274static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, 275 struct ib_port_attr *props) 276{ 277 struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); 278 struct net_device *net_dev; 279 enum ib_mtu mtu; 280 u32 port; 281 int ret; 282 283 port = port_num - 1; 284 285 /* props being zeroed by the caller, avoid zeroing it here */ 286 287 props->max_mtu = hr_dev->caps.max_mtu; 288 props->gid_tbl_len = hr_dev->caps.gid_table_len[port]; 289 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | 290 IB_PORT_VENDOR_CLASS_SUP | 291 IB_PORT_BOOT_MGMT_SUP; 292 props->max_msg_sz = HNS_ROCE_MAX_MSG_LEN; 293 props->pkey_tbl_len = 1; 294 ret = ib_get_eth_speed(ib_dev, port_num, &props->active_speed, 295 &props->active_width); 296 if (ret) 297 ibdev_warn(ib_dev, "failed to get speed, ret = %d.\n", ret); 298 299 net_dev = ib_device_get_netdev(ib_dev, port_num); 300 if (!net_dev) { 301 ibdev_err(ib_dev, "find netdev %u failed!\n", port); 302 return -EINVAL; 303 } 304 305 mtu = iboe_get_mtu(net_dev->mtu); 306 props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; 307 308 dev_put(net_dev); 309 310 ret = hns_roce_get_port_state(hr_dev, port_num, &props->state); 311 if (ret) { 312 ibdev_err(ib_dev, "failed to get port state.\n"); 313 return ret; 314 } 315 316 props->phys_state = props->state == IB_PORT_ACTIVE ? 317 IB_PORT_PHYS_STATE_LINK_UP : 318 IB_PORT_PHYS_STATE_DISABLED; 319 return 0; 320} 321 322static enum rdma_link_layer hns_roce_get_link_layer(struct ib_device *device, 323 u32 port_num) 324{ 325 return IB_LINK_LAYER_ETHERNET; 326} 327 328static int hns_roce_query_pkey(struct ib_device *ib_dev, u32 port, u16 index, 329 u16 *pkey) 330{ 331 if (index > 0) 332 return -EINVAL; 333 334 *pkey = PKEY_ID; 335 336 return 0; 337} 338 339static int hns_roce_modify_device(struct ib_device *ib_dev, int mask, 340 struct ib_device_modify *props) 341{ 342 unsigned long flags; 343 344 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) 345 return -EOPNOTSUPP; 346 347 if (mask & IB_DEVICE_MODIFY_NODE_DESC) { 348 spin_lock_irqsave(&to_hr_dev(ib_dev)->sm_lock, flags); 349 memcpy(ib_dev->node_desc, props->node_desc, NODE_DESC_SIZE); 350 spin_unlock_irqrestore(&to_hr_dev(ib_dev)->sm_lock, flags); 351 } 352 353 return 0; 354} 355 356struct hns_user_mmap_entry * 357hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, 358 size_t length, 359 enum hns_roce_mmap_type mmap_type) 360{ 361 struct hns_user_mmap_entry *entry; 362 int ret; 363 364 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 365 if (!entry) 366 return NULL; 367 368 entry->address = address; 369 entry->mmap_type = mmap_type; 370 371 switch (mmap_type) { 372 /* pgoff 0 must be used by DB for compatibility */ 373 case HNS_ROCE_MMAP_TYPE_DB: 374 ret = rdma_user_mmap_entry_insert_exact( 375 ucontext, &entry->rdma_entry, length, 0); 376 break; 377 case HNS_ROCE_MMAP_TYPE_DWQE: 378 ret = rdma_user_mmap_entry_insert_range( 379 ucontext, &entry->rdma_entry, length, 1, 380 U32_MAX); 381 break; 382 default: 383 ret = -EINVAL; 384 break; 385 } 386 387 if (ret) { 388 kfree(entry); 389 return NULL; 390 } 391 392 return entry; 393} 394 395static void hns_roce_dealloc_uar_entry(struct hns_roce_ucontext *context) 396{ 397 if (context->db_mmap_entry) 398 rdma_user_mmap_entry_remove( 399 &context->db_mmap_entry->rdma_entry); 400} 401 402static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx) 403{ 404 struct hns_roce_ucontext *context = to_hr_ucontext(uctx); 405 u64 address; 406 407 address = context->uar.pfn << PAGE_SHIFT; 408 context->db_mmap_entry = hns_roce_user_mmap_entry_insert( 409 uctx, address, PAGE_SIZE, HNS_ROCE_MMAP_TYPE_DB); 410 if (!context->db_mmap_entry) 411 return -ENOMEM; 412 413 return 0; 414} 415 416static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, 417 struct ib_udata *udata) 418{ 419 struct hns_roce_ucontext *context = to_hr_ucontext(uctx); 420 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); 421 struct hns_roce_ib_alloc_ucontext_resp resp = {}; 422 struct hns_roce_ib_alloc_ucontext ucmd = {}; 423 int ret = -EAGAIN; 424 425 if (!hr_dev->active) 426 goto error_out; 427 428 resp.qp_tab_size = hr_dev->caps.num_qps; 429 resp.srq_tab_size = hr_dev->caps.num_srqs; 430 431 ret = ib_copy_from_udata(&ucmd, udata, 432 min(udata->inlen, sizeof(ucmd))); 433 if (ret) 434 goto error_out; 435 436 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 437 context->config = ucmd.config & HNS_ROCE_EXSGE_FLAGS; 438 439 if (context->config & HNS_ROCE_EXSGE_FLAGS) { 440 resp.config |= HNS_ROCE_RSP_EXSGE_FLAGS; 441 resp.max_inline_data = hr_dev->caps.max_sq_inline; 442 } 443 444 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) { 445 context->config |= ucmd.config & HNS_ROCE_RQ_INLINE_FLAGS; 446 if (context->config & HNS_ROCE_RQ_INLINE_FLAGS) 447 resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS; 448 } 449 450 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQE_INLINE) { 451 context->config |= ucmd.config & HNS_ROCE_CQE_INLINE_FLAGS; 452 if (context->config & HNS_ROCE_CQE_INLINE_FLAGS) 453 resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS; 454 } 455 456 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 457 resp.congest_type = hr_dev->caps.cong_cap; 458 459 ret = hns_roce_uar_alloc(hr_dev, &context->uar); 460 if (ret) 461 goto error_out; 462 463 ret = hns_roce_alloc_uar_entry(uctx); 464 if (ret) 465 goto error_fail_uar_entry; 466 467 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || 468 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) { 469 INIT_LIST_HEAD(&context->page_list); 470 mutex_init(&context->page_mutex); 471 } 472 473 resp.cqe_size = hr_dev->caps.cqe_sz; 474 475 ret = ib_copy_to_udata(udata, &resp, 476 min(udata->outlen, sizeof(resp))); 477 if (ret) 478 goto error_fail_copy_to_udata; 479 480 hns_roce_get_cq_bankid_for_uctx(context); 481 482 return 0; 483 484error_fail_copy_to_udata: 485 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || 486 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) 487 mutex_destroy(&context->page_mutex); 488 hns_roce_dealloc_uar_entry(context); 489 490error_fail_uar_entry: 491 ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); 492 493error_out: 494 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_UCTX_ALLOC_ERR_CNT]); 495 496 return ret; 497} 498 499static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext) 500{ 501 struct hns_roce_ucontext *context = to_hr_ucontext(ibcontext); 502 struct hns_roce_dev *hr_dev = to_hr_dev(ibcontext->device); 503 504 hns_roce_put_cq_bankid_for_uctx(context); 505 506 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB || 507 hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) 508 mutex_destroy(&context->page_mutex); 509 510 hns_roce_dealloc_uar_entry(context); 511 512 ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx); 513} 514 515static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma) 516{ 517 struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); 518 struct rdma_user_mmap_entry *rdma_entry; 519 struct hns_user_mmap_entry *entry; 520 phys_addr_t pfn; 521 pgprot_t prot; 522 int ret; 523 524 if (hr_dev->dis_db) { 525 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]); 526 return -EPERM; 527 } 528 529 rdma_entry = rdma_user_mmap_entry_get_pgoff(uctx, vma->vm_pgoff); 530 if (!rdma_entry) { 531 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]); 532 return -EINVAL; 533 } 534 535 entry = to_hns_mmap(rdma_entry); 536 pfn = entry->address >> PAGE_SHIFT; 537 538 switch (entry->mmap_type) { 539 case HNS_ROCE_MMAP_TYPE_DB: 540 case HNS_ROCE_MMAP_TYPE_DWQE: 541 prot = pgprot_device(vma->vm_page_prot); 542 break; 543 default: 544 ret = -EINVAL; 545 goto out; 546 } 547 548 ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE, 549 prot, rdma_entry); 550 551out: 552 rdma_user_mmap_entry_put(rdma_entry); 553 if (ret) 554 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_MMAP_ERR_CNT]); 555 556 return ret; 557} 558 559static void hns_roce_free_mmap(struct rdma_user_mmap_entry *rdma_entry) 560{ 561 struct hns_user_mmap_entry *entry = to_hns_mmap(rdma_entry); 562 563 kfree(entry); 564} 565 566static int hns_roce_port_immutable(struct ib_device *ib_dev, u32 port_num, 567 struct ib_port_immutable *immutable) 568{ 569 struct ib_port_attr attr; 570 int ret; 571 572 ret = ib_query_port(ib_dev, port_num, &attr); 573 if (ret) 574 return ret; 575 576 immutable->pkey_tbl_len = attr.pkey_tbl_len; 577 immutable->gid_tbl_len = attr.gid_tbl_len; 578 579 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 580 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; 581 if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2) 582 immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 583 584 return 0; 585} 586 587static void hns_roce_disassociate_ucontext(struct ib_ucontext *ibcontext) 588{ 589} 590 591static void hns_roce_get_fw_ver(struct ib_device *device, char *str) 592{ 593 u64 fw_ver = to_hr_dev(device)->caps.fw_ver; 594 unsigned int major, minor, sub_minor; 595 596 major = upper_32_bits(fw_ver); 597 minor = high_16_bits(lower_32_bits(fw_ver)); 598 sub_minor = low_16_bits(fw_ver); 599 600 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%04u", major, minor, 601 sub_minor); 602} 603 604#define HNS_ROCE_HW_CNT(ename, cname) \ 605 [HNS_ROCE_HW_##ename##_CNT].name = cname 606 607static const struct rdma_stat_desc hns_roce_port_stats_descs[] = { 608 HNS_ROCE_HW_CNT(RX_RC_PKT, "rx_rc_pkt"), 609 HNS_ROCE_HW_CNT(RX_UC_PKT, "rx_uc_pkt"), 610 HNS_ROCE_HW_CNT(RX_UD_PKT, "rx_ud_pkt"), 611 HNS_ROCE_HW_CNT(RX_XRC_PKT, "rx_xrc_pkt"), 612 HNS_ROCE_HW_CNT(RX_PKT, "rx_pkt"), 613 HNS_ROCE_HW_CNT(RX_ERR_PKT, "rx_err_pkt"), 614 HNS_ROCE_HW_CNT(RX_CNP_PKT, "rx_cnp_pkt"), 615 HNS_ROCE_HW_CNT(TX_RC_PKT, "tx_rc_pkt"), 616 HNS_ROCE_HW_CNT(TX_UC_PKT, "tx_uc_pkt"), 617 HNS_ROCE_HW_CNT(TX_UD_PKT, "tx_ud_pkt"), 618 HNS_ROCE_HW_CNT(TX_XRC_PKT, "tx_xrc_pkt"), 619 HNS_ROCE_HW_CNT(TX_PKT, "tx_pkt"), 620 HNS_ROCE_HW_CNT(TX_ERR_PKT, "tx_err_pkt"), 621 HNS_ROCE_HW_CNT(TX_CNP_PKT, "tx_cnp_pkt"), 622 HNS_ROCE_HW_CNT(TRP_GET_MPT_ERR_PKT, "trp_get_mpt_err_pkt"), 623 HNS_ROCE_HW_CNT(TRP_GET_IRRL_ERR_PKT, "trp_get_irrl_err_pkt"), 624 HNS_ROCE_HW_CNT(ECN_DB, "ecn_doorbell"), 625 HNS_ROCE_HW_CNT(RX_BUF, "rx_buffer"), 626 HNS_ROCE_HW_CNT(TRP_RX_SOF, "trp_rx_sof"), 627 HNS_ROCE_HW_CNT(CQ_CQE, "cq_cqe"), 628 HNS_ROCE_HW_CNT(CQ_POE, "cq_poe"), 629 HNS_ROCE_HW_CNT(CQ_NOTIFY, "cq_notify"), 630}; 631 632static struct rdma_hw_stats *hns_roce_alloc_hw_port_stats( 633 struct ib_device *device, u32 port_num) 634{ 635 struct hns_roce_dev *hr_dev = to_hr_dev(device); 636 637 if (port_num > hr_dev->caps.num_ports) { 638 ibdev_err(device, "invalid port num.\n"); 639 return NULL; 640 } 641 642 return rdma_alloc_hw_stats_struct(hns_roce_port_stats_descs, 643 ARRAY_SIZE(hns_roce_port_stats_descs), 644 RDMA_HW_STATS_DEFAULT_LIFESPAN); 645} 646 647static int hns_roce_get_hw_stats(struct ib_device *device, 648 struct rdma_hw_stats *stats, 649 u32 port, int index) 650{ 651 struct hns_roce_dev *hr_dev = to_hr_dev(device); 652 int num_counters = HNS_ROCE_HW_CNT_TOTAL; 653 int ret; 654 655 if (port == 0) 656 return 0; 657 658 if (port > hr_dev->caps.num_ports) 659 return -EINVAL; 660 661 ret = hr_dev->hw->query_hw_counter(hr_dev, stats->value, port, 662 &num_counters); 663 if (ret) { 664 ibdev_err(device, "failed to query hw counter, ret = %d\n", 665 ret); 666 return ret; 667 } 668 669 return num_counters; 670} 671 672static void 673 hns_roce_unregister_bond_cleanup(struct hns_roce_dev *hr_dev, 674 struct hns_roce_bond_group *bond_grp) 675{ 676 struct net_device *net_dev; 677 int i; 678 679 /* To avoid the loss of other slave devices when main_hr_dev 680 * is unregistered, re-initialize the remaining slaves before 681 * the bond resources cleanup. 682 */ 683 bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED; 684 for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { 685 net_dev = bond_grp->bond_func_info[i].net_dev; 686 if (net_dev && net_dev != get_hr_netdev(hr_dev, 0)) 687 hns_roce_bond_init_client(bond_grp, i); 688 } 689 690 hns_roce_cleanup_bond(bond_grp); 691} 692 693static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev, 694 bool bond_cleanup) 695{ 696 struct net_device *net_dev = get_hr_netdev(hr_dev, 0); 697 struct hns_roce_ib_iboe *iboe = &hr_dev->iboe; 698 struct hns_roce_bond_group *bond_grp; 699 u8 bus_num = get_hr_bus_num(hr_dev); 700 701 if (bond_cleanup && hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { 702 bond_grp = hns_roce_get_bond_grp(net_dev, bus_num); 703 if (bond_grp) 704 hns_roce_unregister_bond_cleanup(hr_dev, bond_grp); 705 } 706 707 hr_dev->active = false; 708 unregister_netdevice_notifier(&iboe->nb); 709 ib_unregister_device(&hr_dev->ib_dev); 710} 711 712static const struct ib_device_ops hns_roce_dev_ops = { 713 .owner = THIS_MODULE, 714 .driver_id = RDMA_DRIVER_HNS, 715 .uverbs_abi_ver = 1, 716 .uverbs_no_driver_id_binding = 1, 717 718 .get_dev_fw_str = hns_roce_get_fw_ver, 719 .add_gid = hns_roce_add_gid, 720 .alloc_pd = hns_roce_alloc_pd, 721 .alloc_ucontext = hns_roce_alloc_ucontext, 722 .create_ah = hns_roce_create_ah, 723 .create_user_ah = hns_roce_create_ah, 724 .create_cq = hns_roce_create_cq, 725 .create_qp = hns_roce_create_qp, 726 .dealloc_pd = hns_roce_dealloc_pd, 727 .dealloc_ucontext = hns_roce_dealloc_ucontext, 728 .del_gid = hns_roce_del_gid, 729 .dereg_mr = hns_roce_dereg_mr, 730 .destroy_ah = hns_roce_destroy_ah, 731 .destroy_cq = hns_roce_destroy_cq, 732 .disassociate_ucontext = hns_roce_disassociate_ucontext, 733 .get_dma_mr = hns_roce_get_dma_mr, 734 .get_link_layer = hns_roce_get_link_layer, 735 .get_port_immutable = hns_roce_port_immutable, 736 .mmap = hns_roce_mmap, 737 .mmap_free = hns_roce_free_mmap, 738 .modify_device = hns_roce_modify_device, 739 .modify_qp = hns_roce_modify_qp, 740 .query_ah = hns_roce_query_ah, 741 .query_device = hns_roce_query_device, 742 .query_pkey = hns_roce_query_pkey, 743 .query_port = hns_roce_query_port, 744 .reg_user_mr = hns_roce_reg_user_mr, 745 746 INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah), 747 INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq), 748 INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd), 749 INIT_RDMA_OBJ_SIZE(ib_qp, hns_roce_qp, ibqp), 750 INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext), 751}; 752 753static const struct ib_device_ops hns_roce_dev_hw_stats_ops = { 754 .alloc_hw_port_stats = hns_roce_alloc_hw_port_stats, 755 .get_hw_stats = hns_roce_get_hw_stats, 756}; 757 758static const struct ib_device_ops hns_roce_dev_mr_ops = { 759 .rereg_user_mr = hns_roce_rereg_user_mr, 760}; 761 762static const struct ib_device_ops hns_roce_dev_frmr_ops = { 763 .alloc_mr = hns_roce_alloc_mr, 764 .map_mr_sg = hns_roce_map_mr_sg, 765}; 766 767static const struct ib_device_ops hns_roce_dev_srq_ops = { 768 .create_srq = hns_roce_create_srq, 769 .destroy_srq = hns_roce_destroy_srq, 770 771 INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq), 772}; 773 774static const struct ib_device_ops hns_roce_dev_xrcd_ops = { 775 .alloc_xrcd = hns_roce_alloc_xrcd, 776 .dealloc_xrcd = hns_roce_dealloc_xrcd, 777 778 INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd), 779}; 780 781static const struct ib_device_ops hns_roce_dev_restrack_ops = { 782 .fill_res_cq_entry = hns_roce_fill_res_cq_entry, 783 .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, 784 .fill_res_qp_entry = hns_roce_fill_res_qp_entry, 785 .fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw, 786 .fill_res_mr_entry = hns_roce_fill_res_mr_entry, 787 .fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw, 788 .fill_res_srq_entry = hns_roce_fill_res_srq_entry, 789 .fill_res_srq_entry_raw = hns_roce_fill_res_srq_entry_raw, 790}; 791 792static int hns_roce_register_device(struct hns_roce_dev *hr_dev) 793{ 794 struct hns_roce_ib_iboe *iboe = NULL; 795 struct device *dev = hr_dev->dev; 796 struct ib_device *ib_dev = NULL; 797 struct net_device *net_dev; 798 unsigned int i; 799 int ret; 800 801 iboe = &hr_dev->iboe; 802 spin_lock_init(&iboe->lock); 803 804 ib_dev = &hr_dev->ib_dev; 805 806 ib_dev->node_type = RDMA_NODE_IB_CA; 807 ib_dev->dev.parent = dev; 808 809 ib_dev->phys_port_cnt = hr_dev->caps.num_ports; 810 ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; 811 ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; 812 813 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR) 814 ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops); 815 816 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR) 817 ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops); 818 819 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { 820 ib_set_device_ops(ib_dev, &hns_roce_dev_srq_ops); 821 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); 822 } 823 824 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) 825 ib_set_device_ops(ib_dev, &hns_roce_dev_xrcd_ops); 826 827 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09 && 828 !hr_dev->is_vf) 829 ib_set_device_ops(ib_dev, &hns_roce_dev_hw_stats_ops); 830 831 ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); 832 ib_set_device_ops(ib_dev, &hns_roce_dev_ops); 833 ib_set_device_ops(ib_dev, &hns_roce_dev_restrack_ops); 834 835 dma_set_max_seg_size(dev, SZ_2G); 836 837 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) { 838 ret = hns_roce_alloc_bond_grp(hr_dev); 839 if (ret) { 840 dev_err(dev, "failed to alloc bond_grp for bus %u, ret = %d\n", 841 get_hr_bus_num(hr_dev), ret); 842 return ret; 843 } 844 } 845 846 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND && 847 hns_roce_bond_is_active(hr_dev)) { 848 ret = hns_roce_bond_init(hr_dev); 849 if (ret) { 850 dev_err(dev, "failed to init bond!\n"); 851 return ret; 852 } 853 ret = ib_register_device(ib_dev, "hns_bond_%d", dev); 854 } else { 855 for (i = 0; i < hr_dev->caps.num_ports; i++) { 856 net_dev = get_hr_netdev(hr_dev, i); 857 if (!net_dev) 858 continue; 859 860 ret = ib_device_set_netdev(ib_dev, net_dev, i + 1); 861 if (ret) 862 return ret; 863 } 864 ret = ib_register_device(ib_dev, "hns_%d", dev); 865 } 866 if (ret) { 867 dev_err(dev, "ib_register_device failed!\n"); 868 return ret; 869 } 870 871 ret = hns_roce_setup_mtu_mac(hr_dev); 872 if (ret) { 873 dev_err(dev, "setup_mtu_mac failed!\n"); 874 goto error_failed_setup_mtu_mac; 875 } 876 877 iboe->nb.notifier_call = hns_roce_netdev_event; 878 ret = register_netdevice_notifier(&iboe->nb); 879 if (ret) { 880 dev_err(dev, "register_netdevice_notifier failed!\n"); 881 goto error_failed_setup_mtu_mac; 882 } 883 884 hr_dev->active = true; 885 return 0; 886 887error_failed_setup_mtu_mac: 888 ib_unregister_device(ib_dev); 889 890 return ret; 891} 892 893static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) 894{ 895 struct device *dev = hr_dev->dev; 896 int ret; 897 898 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table, 899 HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz, 900 hr_dev->caps.num_mtpts); 901 if (ret) { 902 dev_err(dev, "failed to init MTPT context memory, aborting.\n"); 903 return ret; 904 } 905 906 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table, 907 HEM_TYPE_QPC, hr_dev->caps.qpc_sz, 908 hr_dev->caps.num_qps); 909 if (ret) { 910 dev_err(dev, "failed to init QP context memory, aborting.\n"); 911 goto err_unmap_dmpt; 912 } 913 914 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.irrl_table, 915 HEM_TYPE_IRRL, 916 hr_dev->caps.irrl_entry_sz * 917 hr_dev->caps.max_qp_init_rdma, 918 hr_dev->caps.num_qps); 919 if (ret) { 920 dev_err(dev, "failed to init irrl_table memory, aborting.\n"); 921 goto err_unmap_qp; 922 } 923 924 if (hr_dev->caps.trrl_entry_sz) { 925 ret = hns_roce_init_hem_table(hr_dev, 926 &hr_dev->qp_table.trrl_table, 927 HEM_TYPE_TRRL, 928 hr_dev->caps.trrl_entry_sz * 929 hr_dev->caps.max_qp_dest_rdma, 930 hr_dev->caps.num_qps); 931 if (ret) { 932 dev_err(dev, 933 "failed to init trrl_table memory, aborting.\n"); 934 goto err_unmap_irrl; 935 } 936 } 937 938 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table, 939 HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz, 940 hr_dev->caps.num_cqs); 941 if (ret) { 942 dev_err(dev, "failed to init CQ context memory, aborting.\n"); 943 goto err_unmap_trrl; 944 } 945 946 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { 947 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table, 948 HEM_TYPE_SRQC, 949 hr_dev->caps.srqc_entry_sz, 950 hr_dev->caps.num_srqs); 951 if (ret) { 952 dev_err(dev, 953 "failed to init SRQ context memory, aborting.\n"); 954 goto err_unmap_cq; 955 } 956 } 957 958 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 959 ret = hns_roce_init_hem_table(hr_dev, 960 &hr_dev->qp_table.sccc_table, 961 HEM_TYPE_SCCC, 962 hr_dev->caps.sccc_sz, 963 hr_dev->caps.num_qps); 964 if (ret) { 965 dev_err(dev, 966 "failed to init SCC context memory, aborting.\n"); 967 goto err_unmap_srq; 968 } 969 } 970 971 if (hr_dev->caps.qpc_timer_entry_sz) { 972 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table, 973 HEM_TYPE_QPC_TIMER, 974 hr_dev->caps.qpc_timer_entry_sz, 975 hr_dev->caps.qpc_timer_bt_num); 976 if (ret) { 977 dev_err(dev, 978 "failed to init QPC timer memory, aborting.\n"); 979 goto err_unmap_ctx; 980 } 981 } 982 983 if (hr_dev->caps.cqc_timer_entry_sz) { 984 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table, 985 HEM_TYPE_CQC_TIMER, 986 hr_dev->caps.cqc_timer_entry_sz, 987 hr_dev->caps.cqc_timer_bt_num); 988 if (ret) { 989 dev_err(dev, 990 "failed to init CQC timer memory, aborting.\n"); 991 goto err_unmap_qpc_timer; 992 } 993 } 994 995 if (hr_dev->caps.gmv_entry_sz) { 996 ret = hns_roce_init_hem_table(hr_dev, &hr_dev->gmv_table, 997 HEM_TYPE_GMV, 998 hr_dev->caps.gmv_entry_sz, 999 hr_dev->caps.gmv_entry_num); 1000 if (ret) { 1001 dev_err(dev, 1002 "failed to init gmv table memory, ret = %d\n", 1003 ret); 1004 goto err_unmap_cqc_timer; 1005 } 1006 } 1007 1008 return 0; 1009 1010err_unmap_cqc_timer: 1011 if (hr_dev->caps.cqc_timer_entry_sz) 1012 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cqc_timer_table); 1013 1014err_unmap_qpc_timer: 1015 if (hr_dev->caps.qpc_timer_entry_sz) 1016 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qpc_timer_table); 1017 1018err_unmap_ctx: 1019 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) 1020 hns_roce_cleanup_hem_table(hr_dev, 1021 &hr_dev->qp_table.sccc_table); 1022err_unmap_srq: 1023 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) 1024 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->srq_table.table); 1025 1026err_unmap_cq: 1027 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 1028 1029err_unmap_trrl: 1030 if (hr_dev->caps.trrl_entry_sz) 1031 hns_roce_cleanup_hem_table(hr_dev, 1032 &hr_dev->qp_table.trrl_table); 1033 1034err_unmap_irrl: 1035 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 1036 1037err_unmap_qp: 1038 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 1039 1040err_unmap_dmpt: 1041 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1042 1043 return ret; 1044} 1045 1046static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev) 1047{ 1048 hns_roce_cleanup_bitmap(hr_dev); 1049 mutex_destroy(&hr_dev->pgdir_mutex); 1050} 1051 1052/** 1053 * hns_roce_setup_hca - setup host channel adapter 1054 * @hr_dev: pointer to hns roce device 1055 * Return : int 1056 */ 1057static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) 1058{ 1059 struct device *dev = hr_dev->dev; 1060 int ret; 1061 1062 spin_lock_init(&hr_dev->sm_lock); 1063 1064 INIT_LIST_HEAD(&hr_dev->qp_list); 1065 spin_lock_init(&hr_dev->qp_list_lock); 1066 1067 INIT_LIST_HEAD(&hr_dev->pgdir_list); 1068 mutex_init(&hr_dev->pgdir_mutex); 1069 1070 hns_roce_init_uar_table(hr_dev); 1071 1072 ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar); 1073 if (ret) { 1074 dev_err(dev, "failed to allocate priv_uar.\n"); 1075 goto err_uar_table_free; 1076 } 1077 1078 ret = hns_roce_init_qp_table(hr_dev); 1079 if (ret) { 1080 dev_err(dev, "failed to init qp_table.\n"); 1081 goto err_uar_table_free; 1082 } 1083 1084 hns_roce_init_pd_table(hr_dev); 1085 1086 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC) 1087 hns_roce_init_xrcd_table(hr_dev); 1088 1089 hns_roce_init_mr_table(hr_dev); 1090 1091 hns_roce_init_cq_table(hr_dev); 1092 1093 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) 1094 hns_roce_init_srq_table(hr_dev); 1095 1096 return 0; 1097 1098err_uar_table_free: 1099 ida_destroy(&hr_dev->uar_ida.ida); 1100 mutex_destroy(&hr_dev->pgdir_mutex); 1101 1102 return ret; 1103} 1104 1105static void check_and_get_armed_cq(struct list_head *cq_list, struct ib_cq *cq) 1106{ 1107 struct hns_roce_cq *hr_cq = to_hr_cq(cq); 1108 unsigned long flags; 1109 1110 spin_lock_irqsave(&hr_cq->lock, flags); 1111 if (cq->comp_handler) { 1112 if (!hr_cq->is_armed) { 1113 hr_cq->is_armed = 1; 1114 list_add_tail(&hr_cq->node, cq_list); 1115 } 1116 } 1117 spin_unlock_irqrestore(&hr_cq->lock, flags); 1118} 1119 1120void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev) 1121{ 1122 struct hns_roce_qp *hr_qp; 1123 struct hns_roce_cq *hr_cq; 1124 struct list_head cq_list; 1125 unsigned long flags_qp; 1126 unsigned long flags; 1127 1128 INIT_LIST_HEAD(&cq_list); 1129 1130 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 1131 list_for_each_entry(hr_qp, &hr_dev->qp_list, node) { 1132 spin_lock_irqsave(&hr_qp->sq.lock, flags_qp); 1133 if (hr_qp->sq.tail != hr_qp->sq.head) 1134 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.send_cq); 1135 spin_unlock_irqrestore(&hr_qp->sq.lock, flags_qp); 1136 1137 spin_lock_irqsave(&hr_qp->rq.lock, flags_qp); 1138 if ((!hr_qp->ibqp.srq) && (hr_qp->rq.tail != hr_qp->rq.head)) 1139 check_and_get_armed_cq(&cq_list, hr_qp->ibqp.recv_cq); 1140 spin_unlock_irqrestore(&hr_qp->rq.lock, flags_qp); 1141 } 1142 1143 list_for_each_entry(hr_cq, &cq_list, node) 1144 hns_roce_cq_completion(hr_dev, hr_cq->cqn); 1145 1146 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 1147} 1148 1149static int hns_roce_alloc_dfx_cnt(struct hns_roce_dev *hr_dev) 1150{ 1151 hr_dev->dfx_cnt = kvcalloc(HNS_ROCE_DFX_CNT_TOTAL, sizeof(atomic64_t), 1152 GFP_KERNEL); 1153 if (!hr_dev->dfx_cnt) 1154 return -ENOMEM; 1155 1156 return 0; 1157} 1158 1159static void hns_roce_dealloc_dfx_cnt(struct hns_roce_dev *hr_dev) 1160{ 1161 kvfree(hr_dev->dfx_cnt); 1162} 1163 1164int hns_roce_init(struct hns_roce_dev *hr_dev) 1165{ 1166 struct device *dev = hr_dev->dev; 1167 int ret; 1168 1169 hr_dev->is_reset = false; 1170 1171 ret = hns_roce_alloc_dfx_cnt(hr_dev); 1172 if (ret) 1173 return ret; 1174 1175 if (hr_dev->hw->cmq_init) { 1176 ret = hr_dev->hw->cmq_init(hr_dev); 1177 if (ret) { 1178 dev_err(dev, "init RoCE Command Queue failed!\n"); 1179 goto error_failed_alloc_dfx_cnt; 1180 } 1181 } 1182 1183 ret = hr_dev->hw->hw_profile(hr_dev); 1184 if (ret) { 1185 dev_err(dev, "get RoCE engine profile failed!\n"); 1186 goto error_failed_cmd_init; 1187 } 1188 1189 ret = hns_roce_cmd_init(hr_dev); 1190 if (ret) { 1191 dev_err(dev, "cmd init failed!\n"); 1192 goto error_failed_cmd_init; 1193 } 1194 1195 /* EQ depends on poll mode, event mode depends on EQ */ 1196 ret = hr_dev->hw->init_eq(hr_dev); 1197 if (ret) { 1198 dev_err(dev, "eq init failed!\n"); 1199 goto error_failed_eq_table; 1200 } 1201 1202 if (hr_dev->cmd_mod) { 1203 ret = hns_roce_cmd_use_events(hr_dev); 1204 if (ret) 1205 dev_warn(dev, 1206 "Cmd event mode failed, set back to poll!\n"); 1207 } 1208 1209 ret = hns_roce_init_hem(hr_dev); 1210 if (ret) { 1211 dev_err(dev, "init HEM(Hardware Entry Memory) failed!\n"); 1212 goto error_failed_init_hem; 1213 } 1214 1215 ret = hns_roce_setup_hca(hr_dev); 1216 if (ret) { 1217 dev_err(dev, "setup hca failed!\n"); 1218 goto error_failed_setup_hca; 1219 } 1220 1221 if (hr_dev->hw->hw_init) { 1222 ret = hr_dev->hw->hw_init(hr_dev); 1223 if (ret) { 1224 dev_err(dev, "hw_init failed!\n"); 1225 goto error_failed_engine_init; 1226 } 1227 } 1228 1229 ret = hns_roce_register_device(hr_dev); 1230 if (ret) 1231 goto error_failed_register_device; 1232 1233 hns_roce_register_debugfs(hr_dev); 1234 1235 return 0; 1236 1237error_failed_register_device: 1238 if (hr_dev->hw->hw_exit) 1239 hr_dev->hw->hw_exit(hr_dev); 1240 1241error_failed_engine_init: 1242 hns_roce_teardown_hca(hr_dev); 1243 1244error_failed_setup_hca: 1245 hns_roce_cleanup_hem(hr_dev); 1246 1247error_failed_init_hem: 1248 if (hr_dev->cmd_mod) 1249 hns_roce_cmd_use_polling(hr_dev); 1250 hr_dev->hw->cleanup_eq(hr_dev); 1251 1252error_failed_eq_table: 1253 hns_roce_cmd_cleanup(hr_dev); 1254 1255error_failed_cmd_init: 1256 if (hr_dev->hw->cmq_exit) 1257 hr_dev->hw->cmq_exit(hr_dev); 1258 1259error_failed_alloc_dfx_cnt: 1260 hns_roce_dealloc_dfx_cnt(hr_dev); 1261 1262 return ret; 1263} 1264 1265void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup) 1266{ 1267 hns_roce_unregister_debugfs(hr_dev); 1268 hns_roce_unregister_device(hr_dev, bond_cleanup); 1269 1270 if (hr_dev->hw->hw_exit) 1271 hr_dev->hw->hw_exit(hr_dev); 1272 hns_roce_teardown_hca(hr_dev); 1273 hns_roce_cleanup_hem(hr_dev); 1274 1275 if (hr_dev->cmd_mod) 1276 hns_roce_cmd_use_polling(hr_dev); 1277 1278 hr_dev->hw->cleanup_eq(hr_dev); 1279 hns_roce_cmd_cleanup(hr_dev); 1280 if (hr_dev->hw->cmq_exit) 1281 hr_dev->hw->cmq_exit(hr_dev); 1282 hns_roce_dealloc_dfx_cnt(hr_dev); 1283} 1284 1285MODULE_LICENSE("Dual BSD/GPL"); 1286MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); 1287MODULE_AUTHOR("Nenglong Zhao <zhaonenglong@hisilicon.com>"); 1288MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>"); 1289MODULE_DESCRIPTION("HNS RoCE Driver");
~/
Read-only file preview.