utils.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2015 - 2021 Intel Corporation */
   3#include "main.h"
   4
   5/**
   6 * irdma_arp_table -manage arp table
   7 * @rf: RDMA PCI function
   8 * @ip_addr: ip address for device
   9 * @ipv4: IPv4 flag
  10 * @mac_addr: mac address ptr
  11 * @action: modify, delete or add
  12 */
  13int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
  14		    const u8 *mac_addr, u32 action)
  15{
  16	unsigned long flags;
  17	int arp_index;
  18	u32 ip[4] = {};
  19
  20	if (ipv4)
  21		ip[0] = *ip_addr;
  22	else
  23		memcpy(ip, ip_addr, sizeof(ip));
  24
  25	spin_lock_irqsave(&rf->arp_lock, flags);
  26	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
  27		if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
  28			break;
  29	}
  30
  31	switch (action) {
  32	case IRDMA_ARP_ADD:
  33		if (arp_index != rf->arp_table_size) {
  34			arp_index = -1;
  35			break;
  36		}
  37
  38		arp_index = 0;
  39		if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
  40				     (u32 *)&arp_index, &rf->next_arp_index)) {
  41			arp_index = -1;
  42			break;
  43		}
  44
  45		memcpy(rf->arp_table[arp_index].ip_addr, ip,
  46		       sizeof(rf->arp_table[arp_index].ip_addr));
  47		ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
  48		break;
  49	case IRDMA_ARP_RESOLVE:
  50		if (arp_index == rf->arp_table_size)
  51			arp_index = -1;
  52		break;
  53	case IRDMA_ARP_DELETE:
  54		if (arp_index == rf->arp_table_size) {
  55			arp_index = -1;
  56			break;
  57		}
  58
  59		memset(rf->arp_table[arp_index].ip_addr, 0,
  60		       sizeof(rf->arp_table[arp_index].ip_addr));
  61		eth_zero_addr(rf->arp_table[arp_index].mac_addr);
  62		irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
  63		break;
  64	default:
  65		arp_index = -1;
  66		break;
  67	}
  68
  69	spin_unlock_irqrestore(&rf->arp_lock, flags);
  70	return arp_index;
  71}
  72
  73/**
  74 * irdma_add_arp - add a new arp entry if needed
  75 * @rf: RDMA function
  76 * @ip: IP address
  77 * @ipv4: IPv4 flag
  78 * @mac: MAC address
  79 */
  80int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
  81{
  82	int arpidx;
  83
  84	arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
  85	if (arpidx >= 0) {
  86		if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
  87			return arpidx;
  88
  89		irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
  90				       ipv4, IRDMA_ARP_DELETE);
  91	}
  92
  93	irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
  94
  95	return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
  96}
  97
  98/**
  99 * wr32 - write 32 bits to hw register
 100 * @hw: hardware information including registers
 101 * @reg: register offset
 102 * @val: value to write to register
 103 */
 104inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
 105{
 106	writel(val, hw->hw_addr + reg);
 107}
 108
 109/**
 110 * rd32 - read a 32 bit hw register
 111 * @hw: hardware information including registers
 112 * @reg: register offset
 113 *
 114 * Return value of register content
 115 */
 116inline u32 rd32(struct irdma_hw *hw, u32 reg)
 117{
 118	return readl(hw->hw_addr + reg);
 119}
 120
 121/**
 122 * rd64 - read a 64 bit hw register
 123 * @hw: hardware information including registers
 124 * @reg: register offset
 125 *
 126 * Return value of register content
 127 */
 128inline u64 rd64(struct irdma_hw *hw, u32 reg)
 129{
 130	return readq(hw->hw_addr + reg);
 131}
 132
 133static void irdma_gid_change_event(struct ib_device *ibdev)
 134{
 135	struct ib_event ib_event;
 136
 137	ib_event.event = IB_EVENT_GID_CHANGE;
 138	ib_event.device = ibdev;
 139	ib_event.element.port_num = 1;
 140	ib_dispatch_event(&ib_event);
 141}
 142
 143/**
 144 * irdma_inetaddr_event - system notifier for ipv4 addr events
 145 * @notifier: not used
 146 * @event: event for notifier
 147 * @ptr: if address
 148 */
 149int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
 150			 void *ptr)
 151{
 152	struct in_ifaddr *ifa = ptr;
 153	struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
 154	struct irdma_device *iwdev;
 155	struct ib_device *ibdev;
 156	u32 local_ipaddr;
 157
 158	real_dev = rdma_vlan_dev_real_dev(netdev);
 159	if (!real_dev)
 160		real_dev = netdev;
 161
 162	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 163	if (!ibdev)
 164		return NOTIFY_DONE;
 165
 166	iwdev = to_iwdev(ibdev);
 167	local_ipaddr = ntohl(ifa->ifa_address);
 168	ibdev_dbg(&iwdev->ibdev,
 169		  "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
 170		  event, &local_ipaddr, real_dev->dev_addr);
 171	switch (event) {
 172	case NETDEV_DOWN:
 173		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 174				       &local_ipaddr, true, IRDMA_ARP_DELETE);
 175		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
 176		irdma_gid_change_event(&iwdev->ibdev);
 177		break;
 178	case NETDEV_UP:
 179	case NETDEV_CHANGEADDR:
 180		irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
 181		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
 182		irdma_gid_change_event(&iwdev->ibdev);
 183		break;
 184	default:
 185		break;
 186	}
 187
 188	ib_device_put(ibdev);
 189
 190	return NOTIFY_DONE;
 191}
 192
 193/**
 194 * irdma_inet6addr_event - system notifier for ipv6 addr events
 195 * @notifier: not used
 196 * @event: event for notifier
 197 * @ptr: if address
 198 */
 199int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
 200			  void *ptr)
 201{
 202	struct inet6_ifaddr *ifa = ptr;
 203	struct net_device *real_dev, *netdev = ifa->idev->dev;
 204	struct irdma_device *iwdev;
 205	struct ib_device *ibdev;
 206	u32 local_ipaddr6[4];
 207
 208	real_dev = rdma_vlan_dev_real_dev(netdev);
 209	if (!real_dev)
 210		real_dev = netdev;
 211
 212	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 213	if (!ibdev)
 214		return NOTIFY_DONE;
 215
 216	iwdev = to_iwdev(ibdev);
 217	irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
 218	ibdev_dbg(&iwdev->ibdev,
 219		  "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
 220		  event, local_ipaddr6, real_dev->dev_addr);
 221	switch (event) {
 222	case NETDEV_DOWN:
 223		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
 224				       local_ipaddr6, false, IRDMA_ARP_DELETE);
 225		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
 226		irdma_gid_change_event(&iwdev->ibdev);
 227		break;
 228	case NETDEV_UP:
 229	case NETDEV_CHANGEADDR:
 230		irdma_add_arp(iwdev->rf, local_ipaddr6, false,
 231			      real_dev->dev_addr);
 232		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
 233		irdma_gid_change_event(&iwdev->ibdev);
 234		break;
 235	default:
 236		break;
 237	}
 238
 239	ib_device_put(ibdev);
 240
 241	return NOTIFY_DONE;
 242}
 243
 244/**
 245 * irdma_net_event - system notifier for net events
 246 * @notifier: not used
 247 * @event: event for notifier
 248 * @ptr: neighbor
 249 */
 250int irdma_net_event(struct notifier_block *notifier, unsigned long event,
 251		    void *ptr)
 252{
 253	struct neighbour *neigh = ptr;
 254	struct net_device *real_dev, *netdev;
 255	struct irdma_device *iwdev;
 256	struct ib_device *ibdev;
 257	__be32 *p;
 258	u32 local_ipaddr[4] = {};
 259	bool ipv4 = true;
 260
 261	switch (event) {
 262	case NETEVENT_NEIGH_UPDATE:
 263		netdev = neigh->dev;
 264		real_dev = rdma_vlan_dev_real_dev(netdev);
 265		if (!real_dev)
 266			real_dev = netdev;
 267		ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
 268		if (!ibdev)
 269			return NOTIFY_DONE;
 270
 271		iwdev = to_iwdev(ibdev);
 272		p = (__be32 *)neigh->primary_key;
 273		if (neigh->tbl->family == AF_INET6) {
 274			ipv4 = false;
 275			irdma_copy_ip_ntohl(local_ipaddr, p);
 276		} else {
 277			local_ipaddr[0] = ntohl(*p);
 278		}
 279
 280		ibdev_dbg(&iwdev->ibdev,
 281			  "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
 282			  iwdev->netdev, neigh->nud_state, local_ipaddr,
 283			  neigh->ha);
 284
 285		if (neigh->nud_state & NUD_VALID)
 286			irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
 287
 288		else
 289			irdma_manage_arp_cache(iwdev->rf, neigh->ha,
 290					       local_ipaddr, ipv4,
 291					       IRDMA_ARP_DELETE);
 292		ib_device_put(ibdev);
 293		break;
 294	default:
 295		break;
 296	}
 297
 298	return NOTIFY_DONE;
 299}
 300
 301/**
 302 * irdma_netdevice_event - system notifier for netdev events
 303 * @notifier: not used
 304 * @event: event for notifier
 305 * @ptr: netdev
 306 */
 307int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
 308			  void *ptr)
 309{
 310	struct irdma_device *iwdev;
 311	struct ib_device *ibdev;
 312	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
 313
 314	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
 315	if (!ibdev)
 316		return NOTIFY_DONE;
 317
 318	iwdev = to_iwdev(ibdev);
 319	iwdev->iw_status = 1;
 320	switch (event) {
 321	case NETDEV_DOWN:
 322		iwdev->iw_status = 0;
 323		fallthrough;
 324	default:
 325		break;
 326	}
 327	ib_device_put(ibdev);
 328
 329	return NOTIFY_DONE;
 330}
 331
 332/**
 333 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
 334 * @iwdev: irdma device
 335 */
 336static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
 337{
 338	struct net_device *ip_dev;
 339	struct inet6_dev *idev;
 340	struct inet6_ifaddr *ifp, *tmp;
 341	u32 local_ipaddr6[4];
 342
 343	rcu_read_lock();
 344	for_each_netdev_rcu (&init_net, ip_dev) {
 345		if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
 346		      rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
 347		      ip_dev == iwdev->netdev) &&
 348		      (READ_ONCE(ip_dev->flags) & IFF_UP)) {
 349			idev = __in6_dev_get(ip_dev);
 350			if (!idev) {
 351				ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
 352				break;
 353			}
 354			list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
 355						  if_list) {
 356				ibdev_dbg(&iwdev->ibdev,
 357					  "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
 358					  &ifp->addr,
 359					  rdma_vlan_dev_vlan_id(ip_dev),
 360					  ip_dev->dev_addr);
 361
 362				irdma_copy_ip_ntohl(local_ipaddr6,
 363						    ifp->addr.in6_u.u6_addr32);
 364				irdma_manage_arp_cache(iwdev->rf,
 365						       ip_dev->dev_addr,
 366						       local_ipaddr6, false,
 367						       IRDMA_ARP_ADD);
 368			}
 369		}
 370	}
 371	rcu_read_unlock();
 372}
 373
 374/**
 375 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
 376 * @iwdev: irdma device
 377 */
 378static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
 379{
 380	struct net_device *dev;
 381	struct in_device *idev;
 382	u32 ip_addr;
 383
 384	rcu_read_lock();
 385	for_each_netdev_rcu (&init_net, dev) {
 386		if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
 387		      rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
 388		      dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
 389			const struct in_ifaddr *ifa;
 390
 391			idev = __in_dev_get_rcu(dev);
 392			if (!idev)
 393				continue;
 394
 395			in_dev_for_each_ifa_rcu(ifa, idev) {
 396				ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
 397					  &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
 398					  dev->dev_addr);
 399
 400				ip_addr = ntohl(ifa->ifa_address);
 401				irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
 402						       &ip_addr, true,
 403						       IRDMA_ARP_ADD);
 404			}
 405		}
 406	}
 407	rcu_read_unlock();
 408}
 409
 410/**
 411 * irdma_add_ip - add ip addresses
 412 * @iwdev: irdma device
 413 *
 414 * Add ipv4/ipv6 addresses to the arp cache
 415 */
 416void irdma_add_ip(struct irdma_device *iwdev)
 417{
 418	irdma_add_ipv4_addr(iwdev);
 419	irdma_add_ipv6_addr(iwdev);
 420}
 421
 422/**
 423 * irdma_alloc_and_get_cqp_request - get cqp struct
 424 * @cqp: device cqp ptr
 425 * @wait: cqp to be used in wait mode
 426 */
 427struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
 428							  bool wait)
 429{
 430	struct irdma_cqp_request *cqp_request = NULL;
 431	unsigned long flags;
 432
 433	spin_lock_irqsave(&cqp->req_lock, flags);
 434	if (!list_empty(&cqp->cqp_avail_reqs)) {
 435		cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
 436					       struct irdma_cqp_request, list);
 437		list_del_init(&cqp_request->list);
 438	}
 439	spin_unlock_irqrestore(&cqp->req_lock, flags);
 440	if (!cqp_request) {
 441		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
 442		if (cqp_request) {
 443			cqp_request->dynamic = true;
 444			if (wait)
 445				init_waitqueue_head(&cqp_request->waitq);
 446		}
 447	}
 448	if (!cqp_request) {
 449		ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
 450		return NULL;
 451	}
 452
 453	cqp_request->waiting = wait;
 454	refcount_set(&cqp_request->refcnt, 1);
 455	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
 456	memset(&cqp_request->info, 0, sizeof(cqp_request->info));
 457
 458	return cqp_request;
 459}
 460
 461/**
 462 * irdma_get_cqp_request - increase refcount for cqp_request
 463 * @cqp_request: pointer to cqp_request instance
 464 */
 465static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
 466{
 467	refcount_inc(&cqp_request->refcnt);
 468}
 469
 470/**
 471 * irdma_free_cqp_request - free cqp request
 472 * @cqp: cqp ptr
 473 * @cqp_request: to be put back in cqp list
 474 */
 475void irdma_free_cqp_request(struct irdma_cqp *cqp,
 476			    struct irdma_cqp_request *cqp_request)
 477{
 478	unsigned long flags;
 479
 480	if (cqp_request->dynamic) {
 481		kfree(cqp_request);
 482	} else {
 483		WRITE_ONCE(cqp_request->request_done, false);
 484		cqp_request->callback_fcn = NULL;
 485		cqp_request->waiting = false;
 486		cqp_request->pending = false;
 487
 488		spin_lock_irqsave(&cqp->req_lock, flags);
 489		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
 490		spin_unlock_irqrestore(&cqp->req_lock, flags);
 491	}
 492	wake_up(&cqp->remove_wq);
 493}
 494
 495/**
 496 * irdma_put_cqp_request - dec ref count and free if 0
 497 * @cqp: cqp ptr
 498 * @cqp_request: to be put back in cqp list
 499 */
 500void irdma_put_cqp_request(struct irdma_cqp *cqp,
 501			   struct irdma_cqp_request *cqp_request)
 502{
 503	if (refcount_dec_and_test(&cqp_request->refcnt))
 504		irdma_free_cqp_request(cqp, cqp_request);
 505}
 506
 507/**
 508 * irdma_free_pending_cqp_request -free pending cqp request objs
 509 * @cqp: cqp ptr
 510 * @cqp_request: to be put back in cqp list
 511 */
 512static void
 513irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
 514			       struct irdma_cqp_request *cqp_request)
 515{
 516	if (cqp_request->waiting) {
 517		cqp_request->compl_info.error = true;
 518		WRITE_ONCE(cqp_request->request_done, true);
 519		wake_up(&cqp_request->waitq);
 520	}
 521	wait_event_timeout(cqp->remove_wq,
 522			   refcount_read(&cqp_request->refcnt) == 1, 1000);
 523	irdma_put_cqp_request(cqp, cqp_request);
 524}
 525
 526/**
 527 * irdma_cleanup_deferred_cqp_ops - clean-up cqp with no completions
 528 * @dev: sc_dev
 529 * @cqp: cqp
 530 */
 531static void irdma_cleanup_deferred_cqp_ops(struct irdma_sc_dev *dev,
 532					   struct irdma_cqp *cqp)
 533{
 534	u64 scratch;
 535
 536	/* process all CQP requests with deferred/pending completions */
 537	while ((scratch = irdma_sc_cqp_cleanup_handler(dev)))
 538		irdma_free_pending_cqp_request(cqp, (struct irdma_cqp_request *)
 539						    (uintptr_t)scratch);
 540}
 541
 542/**
 543 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
 544 * completions
 545 * @rf: RDMA PCI function
 546 */
 547void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
 548{
 549	struct irdma_sc_dev *dev = &rf->sc_dev;
 550	struct irdma_cqp *cqp = &rf->cqp;
 551	struct irdma_cqp_request *cqp_request = NULL;
 552	struct cqp_cmds_info *pcmdinfo = NULL;
 553	u32 i, pending_work, wqe_idx;
 554
 555	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3)
 556		irdma_cleanup_deferred_cqp_ops(dev, cqp);
 557	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
 558	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
 559	for (i = 0; i < pending_work; i++) {
 560		cqp_request = (struct irdma_cqp_request *)(unsigned long)
 561				      cqp->scratch_array[wqe_idx];
 562		if (cqp_request)
 563			irdma_free_pending_cqp_request(cqp, cqp_request);
 564		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
 565	}
 566
 567	while (!list_empty(&dev->cqp_cmd_head)) {
 568		pcmdinfo = irdma_remove_cqp_head(dev);
 569		cqp_request =
 570			container_of(pcmdinfo, struct irdma_cqp_request, info);
 571		if (cqp_request)
 572			irdma_free_pending_cqp_request(cqp, cqp_request);
 573	}
 574}
 575
 576static int irdma_get_timeout_threshold(struct irdma_sc_dev *dev)
 577{
 578	u16 time_s = dev->vc_caps.cqp_timeout_s;
 579
 580	if (!time_s)
 581		return CQP_TIMEOUT_THRESHOLD;
 582
 583	return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms;
 584}
 585
 586static int irdma_get_def_timeout_threshold(struct irdma_sc_dev *dev)
 587{
 588	u16 time_s = dev->vc_caps.cqp_def_timeout_s;
 589
 590	if (!time_s)
 591		return CQP_DEF_CMPL_TIMEOUT_THRESHOLD;
 592
 593	return time_s * 1000 / dev->hw_attrs.max_cqp_compl_wait_time_ms;
 594}
 595
 596/**
 597 * irdma_wait_event - wait for completion
 598 * @rf: RDMA PCI function
 599 * @cqp_request: cqp request to wait
 600 */
 601static int irdma_wait_event(struct irdma_pci_f *rf,
 602			    struct irdma_cqp_request *cqp_request)
 603{
 604	struct irdma_cqp_timeout cqp_timeout = {};
 605	int timeout_threshold = irdma_get_timeout_threshold(&rf->sc_dev);
 606	bool cqp_error = false;
 607	int err_code = 0;
 608
 609	cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
 610	do {
 611		irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
 612		if (wait_event_timeout(cqp_request->waitq,
 613				       READ_ONCE(cqp_request->request_done),
 614				       msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
 615			break;
 616
 617		if (cqp_request->pending)
 618			/* There was a deferred or pending completion
 619			 * received for this CQP request, so we need
 620			 * to wait longer than usual.
 621			 */
 622			timeout_threshold =
 623				irdma_get_def_timeout_threshold(&rf->sc_dev);
 624
 625		irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
 626
 627		if (cqp_timeout.count < timeout_threshold)
 628			continue;
 629
 630		if (!rf->reset) {
 631			rf->reset = true;
 632			rf->gen_ops.request_reset(rf);
 633		}
 634		return -ETIMEDOUT;
 635	} while (1);
 636
 637	cqp_error = cqp_request->compl_info.error;
 638	if (cqp_error) {
 639		err_code = -EIO;
 640		if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
 641			if (cqp_request->compl_info.min_err_code == 0x8002)
 642				err_code = -EBUSY;
 643			else if (cqp_request->compl_info.min_err_code == 0x8029) {
 644				if (!rf->reset) {
 645					rf->reset = true;
 646					rf->gen_ops.request_reset(rf);
 647				}
 648			}
 649		}
 650	}
 651
 652	return err_code;
 653}
 654
 655static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
 656	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
 657	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
 658	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
 659	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
 660	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
 661	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
 662	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
 663	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
 664	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
 665	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
 666	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
 667	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
 668	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
 669	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
 670	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
 671	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
 672	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
 673	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
 674	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
 675	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
 676	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
 677	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
 678	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
 679	[IRDMA_OP_RESUME] = "Resume QP Cmd",
 680	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
 681	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
 682	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
 683	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
 684	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
 685	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
 686	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
 687	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
 688	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
 689	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
 690	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
 691	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
 692	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
 693	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
 694	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
 695	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
 696	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
 697	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
 698	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
 699	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
 700	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
 701	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
 702	[IRDMA_OP_SRQ_CREATE] = "Create SRQ Cmd",
 703	[IRDMA_OP_SRQ_MODIFY] = "Modify SRQ Cmd",
 704	[IRDMA_OP_SRQ_DESTROY] = "Destroy SRQ Cmd",
 705};
 706
 707static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
 708	{0xffff, 0x8002, "Invalid State"},
 709	{0xffff, 0x8006, "Flush No Wqe Pending"},
 710	{0xffff, 0x8007, "Modify QP Bad Close"},
 711	{0xffff, 0x8009, "LLP Closed"},
 712	{0xffff, 0x800a, "Reset Not Sent"}
 713};
 714
 715/**
 716 * irdma_cqp_crit_err - check if CQP error is critical
 717 * @dev: pointer to dev structure
 718 * @cqp_cmd: code for last CQP operation
 719 * @maj_err_code: major error code
 720 * @min_err_code: minot error code
 721 */
 722bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
 723			u16 maj_err_code, u16 min_err_code)
 724{
 725	int i;
 726
 727	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
 728		if (maj_err_code == irdma_noncrit_err_list[i].maj &&
 729		    min_err_code == irdma_noncrit_err_list[i].min) {
 730			ibdev_dbg(to_ibdev(dev),
 731				  "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
 732				  irdma_noncrit_err_list[i].desc,
 733				  irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
 734				  min_err_code);
 735			return false;
 736		}
 737	}
 738	return true;
 739}
 740
 741/**
 742 * irdma_handle_cqp_op - process cqp command
 743 * @rf: RDMA PCI function
 744 * @cqp_request: cqp request to process
 745 */
 746int irdma_handle_cqp_op(struct irdma_pci_f *rf,
 747			struct irdma_cqp_request *cqp_request)
 748{
 749	struct irdma_sc_dev *dev = &rf->sc_dev;
 750	struct cqp_cmds_info *info = &cqp_request->info;
 751	int status;
 752	bool put_cqp_request = true;
 753
 754	if (rf->reset)
 755		return -EBUSY;
 756
 757	irdma_get_cqp_request(cqp_request);
 758	status = irdma_process_cqp_cmd(dev, info);
 759	if (status)
 760		goto err;
 761
 762	if (cqp_request->waiting) {
 763		put_cqp_request = false;
 764		status = irdma_wait_event(rf, cqp_request);
 765		if (status)
 766			goto err;
 767	}
 768
 769	return 0;
 770
 771err:
 772	if (irdma_cqp_crit_err(dev, info->cqp_cmd,
 773			       cqp_request->compl_info.maj_err_code,
 774			       cqp_request->compl_info.min_err_code))
 775		ibdev_err(&rf->iwdev->ibdev,
 776			  "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
 777			  irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
 778			  cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
 779			  cqp_request->compl_info.min_err_code);
 780
 781	if (put_cqp_request)
 782		irdma_put_cqp_request(&rf->cqp, cqp_request);
 783
 784	return status;
 785}
 786
 787void irdma_qp_add_ref(struct ib_qp *ibqp)
 788{
 789	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
 790
 791	refcount_inc(&iwqp->refcnt);
 792}
 793
 794void irdma_qp_rem_ref(struct ib_qp *ibqp)
 795{
 796	struct irdma_qp *iwqp = to_iwqp(ibqp);
 797	struct irdma_device *iwdev = iwqp->iwdev;
 798	u32 qp_num;
 799	unsigned long flags;
 800
 801	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
 802	if (!refcount_dec_and_test(&iwqp->refcnt)) {
 803		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 804		return;
 805	}
 806
 807	qp_num = iwqp->ibqp.qp_num;
 808	iwdev->rf->qp_table[qp_num] = NULL;
 809	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
 810	complete(&iwqp->free_qp);
 811}
 812
 813void irdma_cq_add_ref(struct ib_cq *ibcq)
 814{
 815	struct irdma_cq *iwcq = to_iwcq(ibcq);
 816
 817	refcount_inc(&iwcq->refcnt);
 818}
 819
 820void irdma_cq_rem_ref(struct ib_cq *ibcq)
 821{
 822	struct ib_device *ibdev = ibcq->device;
 823	struct irdma_device *iwdev = to_iwdev(ibdev);
 824	struct irdma_cq *iwcq = to_iwcq(ibcq);
 825	unsigned long flags;
 826
 827	spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);
 828	if (!refcount_dec_and_test(&iwcq->refcnt)) {
 829		spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 830		return;
 831	}
 832
 833	iwdev->rf->cq_table[iwcq->cq_num] = NULL;
 834	spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
 835	complete(&iwcq->free_cq);
 836}
 837
 838struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
 839{
 840	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
 841}
 842
 843/**
 844 * irdma_get_qp - get qp address
 845 * @device: iwarp device
 846 * @qpn: qp number
 847 */
 848struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
 849{
 850	struct irdma_device *iwdev = to_iwdev(device);
 851
 852	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
 853		return NULL;
 854
 855	return &iwdev->rf->qp_table[qpn]->ibqp;
 856}
 857
 858/**
 859 * irdma_remove_cqp_head - return head entry and remove
 860 * @dev: device
 861 */
 862void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
 863{
 864	struct list_head *entry;
 865	struct list_head *list = &dev->cqp_cmd_head;
 866
 867	if (list_empty(list))
 868		return NULL;
 869
 870	entry = list->next;
 871	list_del(entry);
 872
 873	return entry;
 874}
 875
 876/**
 877 * irdma_cqp_sds_cmd - create cqp command for sd
 878 * @dev: hardware control device structure
 879 * @sdinfo: information for sd cqp
 880 *
 881 */
 882int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
 883		      struct irdma_update_sds_info *sdinfo)
 884{
 885	struct irdma_cqp_request *cqp_request;
 886	struct cqp_cmds_info *cqp_info;
 887	struct irdma_pci_f *rf = dev_to_rf(dev);
 888	int status;
 889
 890	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
 891	if (!cqp_request)
 892		return -ENOMEM;
 893
 894	cqp_info = &cqp_request->info;
 895	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
 896	       sizeof(cqp_info->in.u.update_pe_sds.info));
 897	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
 898	cqp_info->post_sq = 1;
 899	cqp_info->in.u.update_pe_sds.dev = dev;
 900	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
 901
 902	status = irdma_handle_cqp_op(rf, cqp_request);
 903	irdma_put_cqp_request(&rf->cqp, cqp_request);
 904
 905	return status;
 906}
 907
 908/**
 909 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
 910 * @qp: hardware control qp
 911 * @op: suspend or resume
 912 */
 913int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
 914{
 915	struct irdma_sc_dev *dev = qp->dev;
 916	struct irdma_cqp_request *cqp_request;
 917	struct irdma_sc_cqp *cqp = dev->cqp;
 918	struct cqp_cmds_info *cqp_info;
 919	struct irdma_pci_f *rf = dev_to_rf(dev);
 920	int status;
 921
 922	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
 923	if (!cqp_request)
 924		return -ENOMEM;
 925
 926	cqp_info = &cqp_request->info;
 927	cqp_info->cqp_cmd = op;
 928	cqp_info->in.u.suspend_resume.cqp = cqp;
 929	cqp_info->in.u.suspend_resume.qp = qp;
 930	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
 931
 932	status = irdma_handle_cqp_op(rf, cqp_request);
 933	irdma_put_cqp_request(&rf->cqp, cqp_request);
 934
 935	return status;
 936}
 937
 938/**
 939 * irdma_term_modify_qp - modify qp for term message
 940 * @qp: hardware control qp
 941 * @next_state: qp's next state
 942 * @term: terminate code
 943 * @term_len: length
 944 */
 945void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
 946			  u8 term_len)
 947{
 948	struct irdma_qp *iwqp;
 949
 950	iwqp = qp->qp_uk.back_qp;
 951	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
 952};
 953
 954/**
 955 * irdma_terminate_done - after terminate is completed
 956 * @qp: hardware control qp
 957 * @timeout_occurred: indicates if terminate timer expired
 958 */
 959void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
 960{
 961	struct irdma_qp *iwqp;
 962	u8 hte = 0;
 963	bool first_time;
 964	unsigned long flags;
 965
 966	iwqp = qp->qp_uk.back_qp;
 967	spin_lock_irqsave(&iwqp->lock, flags);
 968	if (iwqp->hte_added) {
 969		iwqp->hte_added = 0;
 970		hte = 1;
 971	}
 972	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
 973	qp->term_flags |= IRDMA_TERM_DONE;
 974	spin_unlock_irqrestore(&iwqp->lock, flags);
 975	if (first_time) {
 976		if (!timeout_occurred)
 977			irdma_terminate_del_timer(qp);
 978
 979		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
 980		irdma_cm_disconn(iwqp);
 981	}
 982}
 983
 984static void irdma_terminate_timeout(struct timer_list *t)
 985{
 986	struct irdma_qp *iwqp = timer_container_of(iwqp, t, terminate_timer);
 987	struct irdma_sc_qp *qp = &iwqp->sc_qp;
 988
 989	irdma_terminate_done(qp, 1);
 990	irdma_qp_rem_ref(&iwqp->ibqp);
 991}
 992
 993/**
 994 * irdma_terminate_start_timer - start terminate timeout
 995 * @qp: hardware control qp
 996 */
 997void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
 998{
 999	struct irdma_qp *iwqp;
1000
1001	iwqp = qp->qp_uk.back_qp;
1002	irdma_qp_add_ref(&iwqp->ibqp);
1003	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
1004	iwqp->terminate_timer.expires = jiffies + HZ;
1005
1006	add_timer(&iwqp->terminate_timer);
1007}
1008
1009/**
1010 * irdma_terminate_del_timer - delete terminate timeout
1011 * @qp: hardware control qp
1012 */
1013void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
1014{
1015	struct irdma_qp *iwqp;
1016	int ret;
1017
1018	iwqp = qp->qp_uk.back_qp;
1019	ret = timer_delete(&iwqp->terminate_timer);
1020	if (ret)
1021		irdma_qp_rem_ref(&iwqp->ibqp);
1022}
1023
1024/**
1025 * irdma_cqp_cq_create_cmd - create a cq for the cqp
1026 * @dev: device pointer
1027 * @cq: pointer to created cq
1028 */
1029int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1030{
1031	struct irdma_pci_f *rf = dev_to_rf(dev);
1032	struct irdma_cqp *iwcqp = &rf->cqp;
1033	struct irdma_cqp_request *cqp_request;
1034	struct cqp_cmds_info *cqp_info;
1035	int status;
1036
1037	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1038	if (!cqp_request)
1039		return -ENOMEM;
1040
1041	cqp_info = &cqp_request->info;
1042	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1043	cqp_info->post_sq = 1;
1044	cqp_info->in.u.cq_create.cq = cq;
1045	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1046
1047	status = irdma_handle_cqp_op(rf, cqp_request);
1048	irdma_put_cqp_request(iwcqp, cqp_request);
1049
1050	return status;
1051}
1052
1053/**
1054 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1055 * @dev: device pointer
1056 * @qp: pointer to created qp
1057 */
1058int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1059{
1060	struct irdma_pci_f *rf = dev_to_rf(dev);
1061	struct irdma_cqp *iwcqp = &rf->cqp;
1062	struct irdma_cqp_request *cqp_request;
1063	struct cqp_cmds_info *cqp_info;
1064	struct irdma_create_qp_info *qp_info;
1065	int status;
1066
1067	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1068	if (!cqp_request)
1069		return -ENOMEM;
1070
1071	cqp_info = &cqp_request->info;
1072	qp_info = &cqp_request->info.in.u.qp_create.info;
1073	qp_info->cq_num_valid = true;
1074	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1075	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1076	cqp_info->post_sq = 1;
1077	cqp_info->in.u.qp_create.qp = qp;
1078	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1079
1080	status = irdma_handle_cqp_op(rf, cqp_request);
1081	irdma_put_cqp_request(iwcqp, cqp_request);
1082
1083	return status;
1084}
1085
1086/**
1087 * irdma_dealloc_push_page - free a push page for qp
1088 * @rf: RDMA PCI function
1089 * @qp: hardware control qp
1090 */
1091static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1092				    struct irdma_sc_qp *qp)
1093{
1094	struct irdma_cqp_request *cqp_request;
1095	struct cqp_cmds_info *cqp_info;
1096	int status;
1097
1098	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1099		return;
1100
1101	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1102	if (!cqp_request)
1103		return;
1104
1105	cqp_info = &cqp_request->info;
1106	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1107	cqp_info->post_sq = 1;
1108	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1109	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1110	cqp_info->in.u.manage_push_page.info.free_page = 1;
1111	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1112	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1113	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1114	status = irdma_handle_cqp_op(rf, cqp_request);
1115	if (!status)
1116		qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1117	irdma_put_cqp_request(&rf->cqp, cqp_request);
1118}
1119
1120static void irdma_free_gsi_qp_rsrc(struct irdma_qp *iwqp, u32 qp_num)
1121{
1122	struct irdma_device *iwdev = iwqp->iwdev;
1123	struct irdma_pci_f *rf = iwdev->rf;
1124	unsigned long flags;
1125
1126	if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_3)
1127		return;
1128
1129	irdma_vchnl_req_del_vport(&rf->sc_dev, iwdev->vport_id, qp_num);
1130
1131	if (qp_num == 1) {
1132		spin_lock_irqsave(&rf->rsrc_lock, flags);
1133		rf->hwqp1_rsvd = false;
1134		spin_unlock_irqrestore(&rf->rsrc_lock, flags);
1135	} else if (qp_num > 2) {
1136		irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1137	}
1138}
1139
1140/**
1141 * irdma_free_qp_rsrc - free up memory resources for qp
1142 * @iwqp: qp ptr (user or kernel)
1143 */
1144void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1145{
1146	struct irdma_device *iwdev = iwqp->iwdev;
1147	struct irdma_pci_f *rf = iwdev->rf;
1148	u32 qp_num = iwqp->sc_qp.qp_uk.qp_id;
1149
1150	irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1151	irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1152	if (iwqp->sc_qp.vsi) {
1153		irdma_qp_rem_qos(&iwqp->sc_qp);
1154		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1155					   iwqp->sc_qp.user_pri);
1156	}
1157
1158	if (iwqp->ibqp.qp_type == IB_QPT_GSI) {
1159		irdma_free_gsi_qp_rsrc(iwqp, qp_num);
1160	} else {
1161		if (qp_num > 2)
1162			irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1163	}
1164	dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1165			  iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1166	iwqp->q2_ctx_mem.va = NULL;
1167	dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1168			  iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1169	iwqp->kqp.dma_mem.va = NULL;
1170	kfree(iwqp->kqp.sq_wrid_mem);
1171	kfree(iwqp->kqp.rq_wrid_mem);
1172}
1173
1174/**
1175 * irdma_srq_wq_destroy - send srq destroy cqp
1176 * @rf: RDMA PCI function
1177 * @srq: hardware control srq
1178 */
1179void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq)
1180{
1181	struct irdma_cqp_request *cqp_request;
1182	struct cqp_cmds_info *cqp_info;
1183
1184	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1185	if (!cqp_request)
1186		return;
1187
1188	cqp_info = &cqp_request->info;
1189	cqp_info->cqp_cmd = IRDMA_OP_SRQ_DESTROY;
1190	cqp_info->post_sq = 1;
1191	cqp_info->in.u.srq_destroy.srq = srq;
1192	cqp_info->in.u.srq_destroy.scratch = (uintptr_t)cqp_request;
1193
1194	irdma_handle_cqp_op(rf, cqp_request);
1195	irdma_put_cqp_request(&rf->cqp, cqp_request);
1196}
1197
1198/**
1199 * irdma_cq_wq_destroy - send cq destroy cqp
1200 * @rf: RDMA PCI function
1201 * @cq: hardware control cq
1202 */
1203void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1204{
1205	struct irdma_cqp_request *cqp_request;
1206	struct cqp_cmds_info *cqp_info;
1207
1208	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1209	if (!cqp_request)
1210		return;
1211
1212	cqp_info = &cqp_request->info;
1213	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1214	cqp_info->post_sq = 1;
1215	cqp_info->in.u.cq_destroy.cq = cq;
1216	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1217
1218	irdma_handle_cqp_op(rf, cqp_request);
1219	irdma_put_cqp_request(&rf->cqp, cqp_request);
1220}
1221
1222/**
1223 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1224 * @cqp_request: modify QP completion
1225 */
1226static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1227{
1228	struct cqp_cmds_info *cqp_info;
1229	struct irdma_qp *iwqp;
1230
1231	cqp_info = &cqp_request->info;
1232	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1233	atomic_dec(&iwqp->hw_mod_qp_pend);
1234	wake_up(&iwqp->mod_qp_waitq);
1235}
1236
1237/**
1238 * irdma_hw_modify_qp - setup cqp for modify qp
1239 * @iwdev: RDMA device
1240 * @iwqp: qp ptr (user or kernel)
1241 * @info: info for modify qp
1242 * @wait: flag to wait or not for modify qp completion
1243 */
1244int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1245		       struct irdma_modify_qp_info *info, bool wait)
1246{
1247	int status;
1248	struct irdma_pci_f *rf = iwdev->rf;
1249	struct irdma_cqp_request *cqp_request;
1250	struct cqp_cmds_info *cqp_info;
1251	struct irdma_modify_qp_info *m_info;
1252
1253	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1254	if (!cqp_request)
1255		return -ENOMEM;
1256
1257	if (!wait) {
1258		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1259		atomic_inc(&iwqp->hw_mod_qp_pend);
1260	}
1261	cqp_info = &cqp_request->info;
1262	m_info = &cqp_info->in.u.qp_modify.info;
1263	memcpy(m_info, info, sizeof(*m_info));
1264	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1265	cqp_info->post_sq = 1;
1266	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1267	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1268	status = irdma_handle_cqp_op(rf, cqp_request);
1269	irdma_put_cqp_request(&rf->cqp, cqp_request);
1270	if (status) {
1271		if (rdma_protocol_roce(&iwdev->ibdev, 1))
1272			return status;
1273
1274		switch (m_info->next_iwarp_state) {
1275			struct irdma_gen_ae_info ae_info;
1276
1277		case IRDMA_QP_STATE_RTS:
1278		case IRDMA_QP_STATE_IDLE:
1279		case IRDMA_QP_STATE_TERMINATE:
1280		case IRDMA_QP_STATE_CLOSING:
1281			if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1282				irdma_send_reset(iwqp->cm_node);
1283			else
1284				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1285			if (!wait) {
1286				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1287				ae_info.ae_src = 0;
1288				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1289			} else {
1290				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1291									      wait);
1292				if (!cqp_request)
1293					return -ENOMEM;
1294
1295				cqp_info = &cqp_request->info;
1296				m_info = &cqp_info->in.u.qp_modify.info;
1297				memcpy(m_info, info, sizeof(*m_info));
1298				cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1299				cqp_info->post_sq = 1;
1300				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1301				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1302				m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1303				m_info->reset_tcp_conn = true;
1304				irdma_handle_cqp_op(rf, cqp_request);
1305				irdma_put_cqp_request(&rf->cqp, cqp_request);
1306			}
1307			break;
1308		case IRDMA_QP_STATE_ERROR:
1309		default:
1310			break;
1311		}
1312	}
1313
1314	return status;
1315}
1316
1317/**
1318 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1319 * @dev: device pointer
1320 * @cq: pointer to cq
1321 */
1322void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1323{
1324	struct irdma_pci_f *rf = dev_to_rf(dev);
1325
1326	irdma_cq_wq_destroy(rf, cq);
1327}
1328
1329/**
1330 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1331 * @dev: device pointer
1332 * @qp: pointer to qp
1333 */
1334int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1335{
1336	struct irdma_pci_f *rf = dev_to_rf(dev);
1337	struct irdma_cqp *iwcqp = &rf->cqp;
1338	struct irdma_cqp_request *cqp_request;
1339	struct cqp_cmds_info *cqp_info;
1340	int status;
1341
1342	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1343	if (!cqp_request)
1344		return -ENOMEM;
1345
1346	cqp_info = &cqp_request->info;
1347	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1348	cqp_info->post_sq = 1;
1349	cqp_info->in.u.qp_destroy.qp = qp;
1350	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1351	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1352
1353	status = irdma_handle_cqp_op(rf, cqp_request);
1354	irdma_put_cqp_request(&rf->cqp, cqp_request);
1355
1356	return status;
1357}
1358
1359/**
1360 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1361 * @dev: hardware control device structure
1362 * @qp: hardware control qp
1363 */
1364void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1365{
1366	struct irdma_gen_ae_info info = {};
1367	struct irdma_pci_f *rf = dev_to_rf(dev);
1368
1369	ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1370	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1371	info.ae_src = IRDMA_AE_SOURCE_RQ;
1372	irdma_gen_ae(rf, qp, &info, false);
1373}
1374
1375/**
1376 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1377 * @addr: address of buffer for crc
1378 * @len: length of buffer
1379 * @val: value to be compared
1380 */
1381int irdma_ieq_check_mpacrc(const void *addr, u32 len, u32 val)
1382{
1383	if ((__force u32)cpu_to_le32(~crc32c(~0, addr, len)) != val)
1384		return -EINVAL;
1385
1386	return 0;
1387}
1388
1389/**
1390 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1391 * @dev: hardware control device structure
1392 * @buf: receive puda buffer on exception q
1393 */
1394struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1395				     struct irdma_puda_buf *buf)
1396{
1397	struct irdma_qp *iwqp;
1398	struct irdma_cm_node *cm_node;
1399	struct irdma_device *iwdev = buf->vsi->back_vsi;
1400	u32 loc_addr[4] = {};
1401	u32 rem_addr[4] = {};
1402	u16 loc_port, rem_port;
1403	struct ipv6hdr *ip6h;
1404	struct iphdr *iph = (struct iphdr *)buf->iph;
1405	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1406
1407	if (iph->version == 4) {
1408		loc_addr[0] = ntohl(iph->daddr);
1409		rem_addr[0] = ntohl(iph->saddr);
1410	} else {
1411		ip6h = (struct ipv6hdr *)buf->iph;
1412		irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1413		irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1414	}
1415	loc_port = ntohs(tcph->dest);
1416	rem_port = ntohs(tcph->source);
1417	cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1418				  loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1419	if (!cm_node)
1420		return NULL;
1421
1422	iwqp = cm_node->iwqp;
1423	irdma_rem_ref_cm_node(cm_node);
1424
1425	return &iwqp->sc_qp;
1426}
1427
1428/**
1429 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1430 * @qp: qp ptr
1431 */
1432void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1433{
1434	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1435	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1436	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1437
1438	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1439	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1440
1441	irdma_send_ack(cm_node);
1442}
1443
1444/**
1445 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1446 * @qp: qp pointer
1447 * @ah_info: AH info pointer
1448 */
1449void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1450				struct irdma_ah_info *ah_info)
1451{
1452	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1453	struct iphdr *iph;
1454	struct ipv6hdr *ip6h;
1455
1456	memset(ah_info, 0, sizeof(*ah_info));
1457	ah_info->do_lpbk = true;
1458	ah_info->vlan_tag = buf->vlan_id;
1459	ah_info->insert_vlan_tag = buf->vlan_valid;
1460	ah_info->ipv4_valid = buf->ipv4;
1461	ah_info->vsi = qp->vsi;
1462
1463	if (buf->smac_valid)
1464		ether_addr_copy(ah_info->mac_addr, buf->smac);
1465
1466	if (buf->ipv4) {
1467		ah_info->ipv4_valid = true;
1468		iph = (struct iphdr *)buf->iph;
1469		ah_info->hop_ttl = iph->ttl;
1470		ah_info->tc_tos = iph->tos;
1471		ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1472		ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1473	} else {
1474		ip6h = (struct ipv6hdr *)buf->iph;
1475		ah_info->hop_ttl = ip6h->hop_limit;
1476		ah_info->tc_tos = ip6h->priority;
1477		irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1478				    ip6h->daddr.in6_u.u6_addr32);
1479		irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1480				    ip6h->saddr.in6_u.u6_addr32);
1481	}
1482
1483	ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1484						ah_info->dest_ip_addr,
1485						ah_info->ipv4_valid,
1486						NULL, IRDMA_ARP_RESOLVE);
1487}
1488
1489/**
1490 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1491 * @buf: puda to update
1492 * @len: length of buffer
1493 * @seqnum: seq number for tcp
1494 */
1495static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1496					     u16 len, u32 seqnum)
1497{
1498	struct tcphdr *tcph;
1499	struct iphdr *iph;
1500	u16 iphlen;
1501	u16 pktsize;
1502	u8 *addr = buf->mem.va;
1503
1504	iphlen = (buf->ipv4) ? 20 : 40;
1505	iph = (struct iphdr *)(addr + buf->maclen);
1506	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1507	pktsize = len + buf->tcphlen + iphlen;
1508	iph->tot_len = htons(pktsize);
1509	tcph->seq = htonl(seqnum);
1510}
1511
1512/**
1513 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1514 * @buf: puda to update
1515 * @len: length of buffer
1516 * @seqnum: seq number for tcp
1517 */
1518void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1519				 u32 seqnum)
1520{
1521	struct tcphdr *tcph;
1522	u8 *addr;
1523
1524	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1525		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1526
1527	addr = buf->mem.va;
1528	tcph = (struct tcphdr *)addr;
1529	tcph->seq = htonl(seqnum);
1530}
1531
1532/**
1533 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1534 * buffer
1535 * @info: to get information
1536 * @buf: puda buffer
1537 */
1538static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1539					  struct irdma_puda_buf *buf)
1540{
1541	struct iphdr *iph;
1542	struct ipv6hdr *ip6h;
1543	struct tcphdr *tcph;
1544	u16 iphlen;
1545	u16 pkt_len;
1546	u8 *mem = buf->mem.va;
1547	struct ethhdr *ethh = buf->mem.va;
1548
1549	if (ethh->h_proto == htons(0x8100)) {
1550		info->vlan_valid = true;
1551		buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1552			       VLAN_VID_MASK;
1553	}
1554
1555	buf->maclen = (info->vlan_valid) ? 18 : 14;
1556	iphlen = (info->l3proto) ? 40 : 20;
1557	buf->ipv4 = (info->l3proto) ? false : true;
1558	buf->iph = mem + buf->maclen;
1559	iph = (struct iphdr *)buf->iph;
1560	buf->tcph = buf->iph + iphlen;
1561	tcph = (struct tcphdr *)buf->tcph;
1562
1563	if (buf->ipv4) {
1564		pkt_len = ntohs(iph->tot_len);
1565	} else {
1566		ip6h = (struct ipv6hdr *)buf->iph;
1567		pkt_len = ntohs(ip6h->payload_len) + iphlen;
1568	}
1569
1570	buf->totallen = pkt_len + buf->maclen;
1571
1572	if (info->payload_len < buf->totallen) {
1573		ibdev_dbg(to_ibdev(buf->vsi->dev),
1574			  "ERR: payload_len = 0x%x totallen expected0x%x\n",
1575			  info->payload_len, buf->totallen);
1576		return -EINVAL;
1577	}
1578
1579	buf->tcphlen = tcph->doff << 2;
1580	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1581	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1582	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1583	buf->seqnum = ntohl(tcph->seq);
1584
1585	return 0;
1586}
1587
1588/**
1589 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1590 * @info: to get information
1591 * @buf: puda buffer
1592 */
1593int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1594			      struct irdma_puda_buf *buf)
1595{
1596	struct tcphdr *tcph;
1597	u32 pkt_len;
1598	u8 *mem;
1599
1600	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1601		return irdma_gen1_puda_get_tcpip_info(info, buf);
1602
1603	mem = buf->mem.va;
1604	buf->vlan_valid = info->vlan_valid;
1605	if (info->vlan_valid)
1606		buf->vlan_id = info->vlan;
1607
1608	buf->ipv4 = info->ipv4;
1609	if (buf->ipv4)
1610		buf->iph = mem + IRDMA_IPV4_PAD;
1611	else
1612		buf->iph = mem;
1613
1614	buf->tcph = mem + IRDMA_TCP_OFFSET;
1615	tcph = (struct tcphdr *)buf->tcph;
1616	pkt_len = info->payload_len;
1617	buf->totallen = pkt_len;
1618	buf->tcphlen = tcph->doff << 2;
1619	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1620	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1621	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1622	buf->seqnum = ntohl(tcph->seq);
1623
1624	if (info->smac_valid) {
1625		ether_addr_copy(buf->smac, info->smac);
1626		buf->smac_valid = true;
1627	}
1628
1629	return 0;
1630}
1631
1632/**
1633 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1634 * @t: timer_list pointer
1635 */
1636static void irdma_hw_stats_timeout(struct timer_list *t)
1637{
1638	struct irdma_vsi_pestat *pf_devstat =
1639		timer_container_of(pf_devstat, t, stats_timer);
1640	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1641
1642	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1643		irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1644	else
1645		irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1646
1647	mod_timer(&pf_devstat->stats_timer,
1648		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1649}
1650
1651/**
1652 * irdma_hw_stats_start_timer - Start periodic stats timer
1653 * @vsi: vsi structure pointer
1654 */
1655void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1656{
1657	struct irdma_vsi_pestat *devstat = vsi->pestat;
1658
1659	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1660	mod_timer(&devstat->stats_timer,
1661		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1662}
1663
1664/**
1665 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1666 * @vsi: pointer to vsi structure
1667 */
1668void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1669{
1670	struct irdma_vsi_pestat *devstat = vsi->pestat;
1671
1672	timer_delete_sync(&devstat->stats_timer);
1673}
1674
1675/**
1676 * irdma_process_stats - Checking for wrap and update stats
1677 * @pestat: stats structure pointer
1678 */
1679static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1680{
1681	sc_vsi_update_stats(pestat->vsi);
1682}
1683
1684/**
1685 * irdma_cqp_gather_stats_gen1 - Gather stats
1686 * @dev: pointer to device structure
1687 * @pestat: statistics structure
1688 */
1689void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1690				 struct irdma_vsi_pestat *pestat)
1691{
1692	struct irdma_gather_stats *gather_stats =
1693		pestat->gather_info.gather_stats_va;
1694	const struct irdma_hw_stat_map *map = dev->hw_stats_map;
1695	u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
1696	u32 stats_inst_offset_32;
1697	u32 stats_inst_offset_64;
1698	u64 new_val;
1699	u16 i;
1700
1701	stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1702				pestat->gather_info.stats_inst_index :
1703				pestat->hw->hmc.hmc_fn_id;
1704	stats_inst_offset_32 *= 4;
1705	stats_inst_offset_64 = stats_inst_offset_32 * 2;
1706
1707	for (i = 0; i < max_stats_idx; i++) {
1708		if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1709			new_val = rd32(dev->hw,
1710				       dev->hw_stats_regs[i] + stats_inst_offset_32);
1711		else
1712			new_val = rd64(dev->hw,
1713				       dev->hw_stats_regs[i] + stats_inst_offset_64);
1714		gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
1715	}
1716
1717	irdma_process_stats(pestat);
1718}
1719
1720/**
1721 * irdma_process_cqp_stats - Checking for wrap and update stats
1722 * @cqp_request: cqp_request structure pointer
1723 */
1724static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1725{
1726	struct irdma_vsi_pestat *pestat = cqp_request->param;
1727
1728	irdma_process_stats(pestat);
1729}
1730
1731/**
1732 * irdma_cqp_gather_stats_cmd - Gather stats
1733 * @dev: pointer to device structure
1734 * @pestat: pointer to stats info
1735 * @wait: flag to wait or not wait for stats
1736 */
1737int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1738			       struct irdma_vsi_pestat *pestat, bool wait)
1739
1740{
1741	struct irdma_pci_f *rf = dev_to_rf(dev);
1742	struct irdma_cqp *iwcqp = &rf->cqp;
1743	struct irdma_cqp_request *cqp_request;
1744	struct cqp_cmds_info *cqp_info;
1745	int status;
1746
1747	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1748	if (!cqp_request)
1749		return -ENOMEM;
1750
1751	cqp_info = &cqp_request->info;
1752	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1753	cqp_info->post_sq = 1;
1754	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1755	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1756	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1757	cqp_request->param = pestat;
1758	if (!wait)
1759		cqp_request->callback_fcn = irdma_process_cqp_stats;
1760	status = irdma_handle_cqp_op(rf, cqp_request);
1761	if (wait)
1762		irdma_process_stats(pestat);
1763	irdma_put_cqp_request(&rf->cqp, cqp_request);
1764
1765	return status;
1766}
1767
1768/**
1769 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1770 * @vsi: pointer to vsi structure
1771 * @cmd: command to allocate or free
1772 * @stats_info: pointer to allocate stats info
1773 */
1774int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1775			     struct irdma_stats_inst_info *stats_info)
1776{
1777	struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1778	struct irdma_cqp *iwcqp = &rf->cqp;
1779	struct irdma_cqp_request *cqp_request;
1780	struct cqp_cmds_info *cqp_info;
1781	int status;
1782	bool wait = false;
1783
1784	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1785		wait = true;
1786	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1787	if (!cqp_request)
1788		return -ENOMEM;
1789
1790	cqp_info = &cqp_request->info;
1791	cqp_info->cqp_cmd = cmd;
1792	cqp_info->post_sq = 1;
1793	cqp_info->in.u.stats_manage.info = *stats_info;
1794	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1795	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1796	status = irdma_handle_cqp_op(rf, cqp_request);
1797	if (wait)
1798		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1799	irdma_put_cqp_request(iwcqp, cqp_request);
1800
1801	return status;
1802}
1803
1804/**
1805 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1806 * @dev: pointer to device info
1807 * @sc_ceq: pointer to ceq structure
1808 * @op: Create or Destroy
1809 */
1810int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1811		      u8 op)
1812{
1813	struct irdma_cqp_request *cqp_request;
1814	struct cqp_cmds_info *cqp_info;
1815	struct irdma_pci_f *rf = dev_to_rf(dev);
1816	int status;
1817
1818	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1819	if (!cqp_request)
1820		return -ENOMEM;
1821
1822	cqp_info = &cqp_request->info;
1823	cqp_info->post_sq = 1;
1824	cqp_info->cqp_cmd = op;
1825	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1826	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1827
1828	status = irdma_handle_cqp_op(rf, cqp_request);
1829	irdma_put_cqp_request(&rf->cqp, cqp_request);
1830
1831	return status;
1832}
1833
1834/**
1835 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1836 * @dev: pointer to device info
1837 * @sc_aeq: pointer to aeq structure
1838 * @op: Create or Destroy
1839 */
1840int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1841		      u8 op)
1842{
1843	struct irdma_cqp_request *cqp_request;
1844	struct cqp_cmds_info *cqp_info;
1845	struct irdma_pci_f *rf = dev_to_rf(dev);
1846	int status;
1847
1848	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1849	if (!cqp_request)
1850		return -ENOMEM;
1851
1852	cqp_info = &cqp_request->info;
1853	cqp_info->post_sq = 1;
1854	cqp_info->cqp_cmd = op;
1855	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1856	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1857
1858	status = irdma_handle_cqp_op(rf, cqp_request);
1859	irdma_put_cqp_request(&rf->cqp, cqp_request);
1860
1861	return status;
1862}
1863
1864/**
1865 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1866 * @dev: pointer to device structure
1867 * @cmd: Add, modify or delete
1868 * @node_info: pointer to ws node info
1869 */
1870int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1871			  struct irdma_ws_node_info *node_info)
1872{
1873	struct irdma_pci_f *rf = dev_to_rf(dev);
1874	struct irdma_cqp *iwcqp = &rf->cqp;
1875	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1876	struct irdma_cqp_request *cqp_request;
1877	struct cqp_cmds_info *cqp_info;
1878	int status;
1879	bool poll;
1880
1881	if (!rf->sc_dev.ceq_valid)
1882		poll = true;
1883	else
1884		poll = false;
1885
1886	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1887	if (!cqp_request)
1888		return -ENOMEM;
1889
1890	cqp_info = &cqp_request->info;
1891	cqp_info->cqp_cmd = cmd;
1892	cqp_info->post_sq = 1;
1893	cqp_info->in.u.ws_node.info = *node_info;
1894	cqp_info->in.u.ws_node.cqp = cqp;
1895	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1896	status = irdma_handle_cqp_op(rf, cqp_request);
1897	if (status)
1898		goto exit;
1899
1900	if (poll) {
1901		struct irdma_ccq_cqe_info compl_info;
1902
1903		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1904						       &compl_info);
1905		node_info->qs_handle = compl_info.op_ret_val;
1906		ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
1907			  compl_info.op_code, compl_info.op_ret_val);
1908	} else {
1909		node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1910	}
1911
1912exit:
1913	irdma_put_cqp_request(&rf->cqp, cqp_request);
1914
1915	return status;
1916}
1917
1918/**
1919 * irdma_ah_cqp_op - perform an AH cqp operation
1920 * @rf: RDMA PCI function
1921 * @sc_ah: address handle
1922 * @cmd: AH operation
1923 * @wait: wait if true
1924 * @callback_fcn: Callback function on CQP op completion
1925 * @cb_param: parameter for callback function
1926 *
1927 * returns errno
1928 */
1929int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1930		    bool wait,
1931		    void (*callback_fcn)(struct irdma_cqp_request *),
1932		    void *cb_param)
1933{
1934	struct irdma_cqp_request *cqp_request;
1935	struct cqp_cmds_info *cqp_info;
1936	int status;
1937
1938	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1939		return -EINVAL;
1940
1941	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1942	if (!cqp_request)
1943		return -ENOMEM;
1944
1945	cqp_info = &cqp_request->info;
1946	cqp_info->cqp_cmd = cmd;
1947	cqp_info->post_sq = 1;
1948	if (cmd == IRDMA_OP_AH_CREATE) {
1949		cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1950		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1951		cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1952	} else if (cmd == IRDMA_OP_AH_DESTROY) {
1953		cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1954		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1955		cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1956	}
1957
1958	if (!wait) {
1959		cqp_request->callback_fcn = callback_fcn;
1960		cqp_request->param = cb_param;
1961	}
1962	status = irdma_handle_cqp_op(rf, cqp_request);
1963	irdma_put_cqp_request(&rf->cqp, cqp_request);
1964
1965	if (status)
1966		return -ENOMEM;
1967
1968	if (wait)
1969		sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
1970
1971	return 0;
1972}
1973
1974/**
1975 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1976 * @cqp_request: pointer to cqp_request of create AH
1977 */
1978static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1979{
1980	struct irdma_sc_qp *qp = cqp_request->param;
1981	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
1982	unsigned long flags;
1983
1984	spin_lock_irqsave(&qp->pfpdu.lock, flags);
1985	if (!cqp_request->compl_info.op_ret_val) {
1986		sc_ah->ah_info.ah_valid = true;
1987		irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
1988	} else {
1989		sc_ah->ah_info.ah_valid = false;
1990		irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
1991	}
1992	spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
1993}
1994
1995/**
1996 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1997 * @cqp_request: pointer to cqp_request of create AH
1998 */
1999static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2000{
2001	struct irdma_cm_node *cm_node = cqp_request->param;
2002	struct irdma_sc_ah *sc_ah = cm_node->ah;
2003
2004	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2005	irdma_add_conn_est_qh(cm_node);
2006}
2007
2008/**
2009 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2010 * @dev: device pointer
2011 * @ah_info: Address handle info
2012 * @wait: When true will wait for operation to complete
2013 * @type: ILQ/IEQ
2014 * @cb_param: Callback param when not waiting
2015 * @ah_ret: Returned pointer to address handle if created
2016 *
2017 */
2018int irdma_puda_create_ah(struct irdma_sc_dev *dev,
2019			 struct irdma_ah_info *ah_info, bool wait,
2020			 enum puda_rsrc_type type, void *cb_param,
2021			 struct irdma_sc_ah **ah_ret)
2022{
2023	struct irdma_sc_ah *ah;
2024	struct irdma_pci_f *rf = dev_to_rf(dev);
2025	int err;
2026
2027	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2028	*ah_ret = ah;
2029	if (!ah)
2030		return -ENOMEM;
2031
2032	err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2033			       &ah_info->ah_idx, &rf->next_ah);
2034	if (err)
2035		goto err_free;
2036
2037	ah->dev = dev;
2038	ah->ah_info = *ah_info;
2039
2040	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2041		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2042				      irdma_ilq_ah_cb, cb_param);
2043	else
2044		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2045				      irdma_ieq_ah_cb, cb_param);
2046
2047	if (err)
2048		goto error;
2049	return 0;
2050
2051error:
2052	irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2053err_free:
2054	kfree(ah);
2055	*ah_ret = NULL;
2056	return -ENOMEM;
2057}
2058
2059/**
2060 * irdma_puda_free_ah - free a puda address handle
2061 * @dev: device pointer
2062 * @ah: The address handle to free
2063 */
2064void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2065{
2066	struct irdma_pci_f *rf = dev_to_rf(dev);
2067
2068	if (!ah)
2069		return;
2070
2071	if (ah->ah_info.ah_valid) {
2072		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2073		irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2074	}
2075
2076	kfree(ah);
2077}
2078
2079/**
2080 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2081 * @cqp_request: pointer to cqp_request of create AH
2082 */
2083void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2084{
2085	struct irdma_sc_ah *sc_ah = cqp_request->param;
2086
2087	if (!cqp_request->compl_info.op_ret_val)
2088		sc_ah->ah_info.ah_valid = true;
2089	else
2090		sc_ah->ah_info.ah_valid = false;
2091}
2092
2093/**
2094 * irdma_prm_add_pble_mem - add moemory to pble resources
2095 * @pprm: pble resource manager
2096 * @pchunk: chunk of memory to add
2097 */
2098int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2099			   struct irdma_chunk *pchunk)
2100{
2101	u64 sizeofbitmap;
2102
2103	if (pchunk->size & 0xfff)
2104		return -EINVAL;
2105
2106	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2107
2108	pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2109	if (!pchunk->bitmapbuf)
2110		return -ENOMEM;
2111
2112	pchunk->sizeofbitmap = sizeofbitmap;
2113	/* each pble is 8 bytes hence shift by 3 */
2114	pprm->total_pble_alloc += pchunk->size >> 3;
2115	pprm->free_pble_cnt += pchunk->size >> 3;
2116
2117	return 0;
2118}
2119
2120/**
2121 * irdma_prm_get_pbles - get pble's from prm
2122 * @pprm: pble resource manager
2123 * @chunkinfo: nformation about chunk where pble's were acquired
2124 * @mem_size: size of pble memory needed
2125 * @vaddr: returns virtual address of pble memory
2126 * @fpm_addr: returns fpm address of pble memory
2127 */
2128int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2129			struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2130			u64 **vaddr, u64 *fpm_addr)
2131{
2132	u64 bits_needed;
2133	u64 bit_idx = PBLE_INVALID_IDX;
2134	struct irdma_chunk *pchunk = NULL;
2135	struct list_head *chunk_entry = pprm->clist.next;
2136	u32 offset;
2137	unsigned long flags;
2138	*vaddr = NULL;
2139	*fpm_addr = 0;
2140
2141	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2142
2143	spin_lock_irqsave(&pprm->prm_lock, flags);
2144	while (chunk_entry != &pprm->clist) {
2145		pchunk = (struct irdma_chunk *)chunk_entry;
2146		bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2147						     pchunk->sizeofbitmap, 0,
2148						     bits_needed, 0);
2149		if (bit_idx < pchunk->sizeofbitmap)
2150			break;
2151
2152		/* list.next used macro */
2153		chunk_entry = pchunk->list.next;
2154	}
2155
2156	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2157		spin_unlock_irqrestore(&pprm->prm_lock, flags);
2158		return -ENOMEM;
2159	}
2160
2161	bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2162	offset = bit_idx << pprm->pble_shift;
2163	*vaddr = pchunk->vaddr + offset;
2164	*fpm_addr = pchunk->fpm_addr + offset;
2165
2166	chunkinfo->pchunk = pchunk;
2167	chunkinfo->bit_idx = bit_idx;
2168	chunkinfo->bits_used = bits_needed;
2169	/* 3 is sizeof pble divide */
2170	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2171	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2172
2173	return 0;
2174}
2175
2176/**
2177 * irdma_prm_return_pbles - return pbles back to prm
2178 * @pprm: pble resource manager
2179 * @chunkinfo: chunk where pble's were acquired and to be freed
2180 */
2181void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2182			    struct irdma_pble_chunkinfo *chunkinfo)
2183{
2184	unsigned long flags;
2185
2186	spin_lock_irqsave(&pprm->prm_lock, flags);
2187	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2188	bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2189		     chunkinfo->bits_used);
2190	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2191}
2192
2193int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2194			   u32 pg_cnt)
2195{
2196	struct page *vm_page;
2197	int i;
2198	u8 *addr;
2199
2200	addr = (u8 *)(uintptr_t)va;
2201	for (i = 0; i < pg_cnt; i++) {
2202		vm_page = vmalloc_to_page(addr);
2203		if (!vm_page)
2204			goto err;
2205
2206		pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2207					 DMA_BIDIRECTIONAL);
2208		if (dma_mapping_error(hw->device, pg_dma[i]))
2209			goto err;
2210
2211		addr += PAGE_SIZE;
2212	}
2213
2214	return 0;
2215
2216err:
2217	irdma_unmap_vm_page_list(hw, pg_dma, i);
2218	return -ENOMEM;
2219}
2220
2221void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2222{
2223	int i;
2224
2225	for (i = 0; i < pg_cnt; i++)
2226		dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2227}
2228
2229/**
2230 * irdma_pble_free_paged_mem - free virtual paged memory
2231 * @chunk: chunk to free with paged memory
2232 */
2233void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2234{
2235	if (!chunk->pg_cnt)
2236		goto done;
2237
2238	irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2239				 chunk->pg_cnt);
2240
2241done:
2242	kfree(chunk->dmainfo.dmaaddrs);
2243	chunk->dmainfo.dmaaddrs = NULL;
2244	vfree(chunk->vaddr);
2245	chunk->vaddr = NULL;
2246	chunk->type = 0;
2247}
2248
2249/**
2250 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2251 * @chunk: chunk to add for paged memory
2252 * @pg_cnt: number of pages needed
2253 */
2254int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2255{
2256	u32 size;
2257	void *va;
2258
2259	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2260	if (!chunk->dmainfo.dmaaddrs)
2261		return -ENOMEM;
2262
2263	size = PAGE_SIZE * pg_cnt;
2264	va = vmalloc(size);
2265	if (!va)
2266		goto err;
2267
2268	if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2269				   pg_cnt)) {
2270		vfree(va);
2271		goto err;
2272	}
2273	chunk->vaddr = va;
2274	chunk->size = size;
2275	chunk->pg_cnt = pg_cnt;
2276	chunk->type = PBLE_SD_PAGED;
2277
2278	return 0;
2279err:
2280	kfree(chunk->dmainfo.dmaaddrs);
2281	chunk->dmainfo.dmaaddrs = NULL;
2282
2283	return -ENOMEM;
2284}
2285
2286/**
2287 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2288 * @dev: device pointer
2289 */
2290u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2291{
2292	struct irdma_pci_f *rf = dev_to_rf(dev);
2293	u32 next = 1;
2294	u32 node_id;
2295
2296	if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2297			     &node_id, &next))
2298		return IRDMA_WS_NODE_INVALID;
2299
2300	return (u16)node_id;
2301}
2302
2303/**
2304 * irdma_free_ws_node_id - Free a tx scheduler node ID
2305 * @dev: device pointer
2306 * @node_id: Work scheduler node ID
2307 */
2308void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2309{
2310	struct irdma_pci_f *rf = dev_to_rf(dev);
2311
2312	irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2313}
2314
2315/**
2316 * irdma_modify_qp_to_err - Modify a QP to error
2317 * @sc_qp: qp structure
2318 */
2319void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2320{
2321	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2322	struct ib_qp_attr attr;
2323
2324	attr.qp_state = IB_QPS_ERR;
2325
2326	if (rdma_protocol_roce(qp->ibqp.device, 1))
2327		irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2328	else
2329		irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2330}
2331
2332void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2333{
2334	struct ib_event ibevent;
2335
2336	if (!iwqp->ibqp.event_handler)
2337		return;
2338
2339	switch (event) {
2340	case IRDMA_QP_EVENT_CATASTROPHIC:
2341		ibevent.event = IB_EVENT_QP_FATAL;
2342		break;
2343	case IRDMA_QP_EVENT_ACCESS_ERR:
2344		ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2345		break;
2346	case IRDMA_QP_EVENT_REQ_ERR:
2347		ibevent.event = IB_EVENT_QP_REQ_ERR;
2348		break;
2349	}
2350	ibevent.device = iwqp->ibqp.device;
2351	ibevent.element.qp = &iwqp->ibqp;
2352	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2353}
2354
2355void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2356{
2357	struct irdma_cmpl_gen *cmpl_node;
2358	struct list_head *tmp_node, *list_node;
2359
2360	list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2361		cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2362		list_del(&cmpl_node->list);
2363		kfree(cmpl_node);
2364	}
2365}
2366
2367int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2368{
2369	struct irdma_cmpl_gen *cmpl;
2370
2371	if (list_empty(&iwcq->cmpl_generated))
2372		return -ENOENT;
2373	cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2374	list_del(&cmpl->list);
2375	memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2376	kfree(cmpl);
2377
2378	ibdev_dbg(iwcq->ibcq.device,
2379		  "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2380		  __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2381		  cq_poll_info->wr_id);
2382
2383	return 0;
2384}
2385
2386/**
2387 * irdma_set_cpi_common_values - fill in values for polling info struct
2388 * @cpi: resulting structure of cq_poll_info type
2389 * @qp: QPair
2390 * @qp_num: id of the QP
2391 */
2392static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2393					struct irdma_qp_uk *qp, u32 qp_num)
2394{
2395	cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2396	cpi->error = true;
2397	cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2398	cpi->minor_err = FLUSH_GENERAL_ERR;
2399	cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2400	cpi->qp_id = qp_num;
2401}
2402
2403static inline void irdma_comp_handler(struct irdma_cq *cq)
2404{
2405	if (!cq->ibcq.comp_handler)
2406		return;
2407	if (atomic_cmpxchg(&cq->armed, 1, 0))
2408		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2409}
2410
2411void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2412{
2413	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2414	struct irdma_ring *sq_ring = &qp->sq_ring;
2415	struct irdma_ring *rq_ring = &qp->rq_ring;
2416	struct irdma_cq *iwscq = iwqp->iwscq;
2417	struct irdma_cq *iwrcq = iwqp->iwrcq;
2418	struct irdma_cmpl_gen *cmpl;
2419	__le64 *sw_wqe;
2420	u64 wqe_qword;
2421	u32 wqe_idx;
2422	bool compl_generated = false;
2423	unsigned long flags1;
2424
2425	spin_lock_irqsave(&iwscq->lock, flags1);
2426	if (irdma_uk_cq_empty(&iwscq->sc_cq.cq_uk)) {
2427		unsigned long flags2;
2428
2429		spin_lock_irqsave(&iwqp->lock, flags2);
2430		while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2431			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2432			if (!cmpl) {
2433				spin_unlock_irqrestore(&iwqp->lock, flags2);
2434				spin_unlock_irqrestore(&iwscq->lock, flags1);
2435				return;
2436			}
2437
2438			wqe_idx = sq_ring->tail;
2439			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2440
2441			cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2442			sw_wqe = qp->sq_base[wqe_idx].elem;
2443			get_64bit_val(sw_wqe, 24, &wqe_qword);
2444			cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2445			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2446			/* remove the SQ WR by moving SQ tail*/
2447			IRDMA_RING_SET_TAIL(*sq_ring,
2448				sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2449			if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2450				kfree(cmpl);
2451				continue;
2452			}
2453			ibdev_dbg(iwscq->ibcq.device,
2454				  "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2455				  __func__, cmpl->cpi.wr_id, qp->qp_id);
2456			list_add_tail(&cmpl->list, &iwscq->cmpl_generated);
2457			compl_generated = true;
2458		}
2459		spin_unlock_irqrestore(&iwqp->lock, flags2);
2460		spin_unlock_irqrestore(&iwscq->lock, flags1);
2461		if (compl_generated)
2462			irdma_comp_handler(iwscq);
2463	} else {
2464		spin_unlock_irqrestore(&iwscq->lock, flags1);
2465		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2466				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2467	}
2468
2469	spin_lock_irqsave(&iwrcq->lock, flags1);
2470	if (irdma_uk_cq_empty(&iwrcq->sc_cq.cq_uk)) {
2471		unsigned long flags2;
2472
2473		spin_lock_irqsave(&iwqp->lock, flags2);
2474		while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2475			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2476			if (!cmpl) {
2477				spin_unlock_irqrestore(&iwqp->lock, flags2);
2478				spin_unlock_irqrestore(&iwrcq->lock, flags1);
2479				return;
2480			}
2481
2482			wqe_idx = rq_ring->tail;
2483			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2484
2485			cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2486			cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2487			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2488			/* remove the RQ WR by moving RQ tail */
2489			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2490			ibdev_dbg(iwrcq->ibcq.device,
2491				  "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2492				  __func__, cmpl->cpi.wr_id, qp->qp_id,
2493				  wqe_idx);
2494			list_add_tail(&cmpl->list, &iwrcq->cmpl_generated);
2495
2496			compl_generated = true;
2497		}
2498		spin_unlock_irqrestore(&iwqp->lock, flags2);
2499		spin_unlock_irqrestore(&iwrcq->lock, flags1);
2500		if (compl_generated)
2501			irdma_comp_handler(iwrcq);
2502	} else {
2503		spin_unlock_irqrestore(&iwrcq->lock, flags1);
2504		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2505				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2506	}
2507}