cxgbit_cm.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2016 Chelsio Communications, Inc.
   4 */
   5
   6#include <linux/module.h>
   7#include <linux/list.h>
   8#include <linux/workqueue.h>
   9#include <linux/skbuff.h>
  10#include <linux/timer.h>
  11#include <linux/notifier.h>
  12#include <linux/inetdevice.h>
  13#include <linux/ip.h>
  14#include <linux/tcp.h>
  15#include <linux/if_vlan.h>
  16
  17#include <net/neighbour.h>
  18#include <net/netevent.h>
  19#include <net/route.h>
  20#include <net/tcp.h>
  21#include <net/ip6_route.h>
  22#include <net/addrconf.h>
  23
  24#include <libcxgb_cm.h>
  25#include "cxgbit.h"
  26#include "clip_tbl.h"
  27
  28static void cxgbit_init_wr_wait(struct cxgbit_wr_wait *wr_waitp)
  29{
  30	wr_waitp->ret = 0;
  31	reinit_completion(&wr_waitp->completion);
  32}
  33
  34static void
  35cxgbit_wake_up(struct cxgbit_wr_wait *wr_waitp, const char *func, u8 ret)
  36{
  37	if (ret == CPL_ERR_NONE)
  38		wr_waitp->ret = 0;
  39	else
  40		wr_waitp->ret = -EIO;
  41
  42	if (wr_waitp->ret)
  43		pr_err("%s: err:%u", func, ret);
  44
  45	complete(&wr_waitp->completion);
  46}
  47
  48static int
  49cxgbit_wait_for_reply(struct cxgbit_device *cdev,
  50		      struct cxgbit_wr_wait *wr_waitp, u32 tid, u32 timeout,
  51		      const char *func)
  52{
  53	int ret;
  54
  55	if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
  56		wr_waitp->ret = -EIO;
  57		goto out;
  58	}
  59
  60	ret = wait_for_completion_timeout(&wr_waitp->completion, timeout * HZ);
  61	if (!ret) {
  62		pr_info("%s - Device %s not responding tid %u\n",
  63			func, pci_name(cdev->lldi.pdev), tid);
  64		wr_waitp->ret = -ETIMEDOUT;
  65	}
  66out:
  67	if (wr_waitp->ret)
  68		pr_info("%s: FW reply %d tid %u\n",
  69			pci_name(cdev->lldi.pdev), wr_waitp->ret, tid);
  70	return wr_waitp->ret;
  71}
  72
  73static int cxgbit_np_hashfn(const struct cxgbit_np *cnp)
  74{
  75	return ((unsigned long)cnp >> 10) & (NP_INFO_HASH_SIZE - 1);
  76}
  77
  78static struct np_info *
  79cxgbit_np_hash_add(struct cxgbit_device *cdev, struct cxgbit_np *cnp,
  80		   unsigned int stid)
  81{
  82	struct np_info *p = kzalloc(sizeof(*p), GFP_KERNEL);
  83
  84	if (p) {
  85		int bucket = cxgbit_np_hashfn(cnp);
  86
  87		p->cnp = cnp;
  88		p->stid = stid;
  89		spin_lock(&cdev->np_lock);
  90		p->next = cdev->np_hash_tab[bucket];
  91		cdev->np_hash_tab[bucket] = p;
  92		spin_unlock(&cdev->np_lock);
  93	}
  94
  95	return p;
  96}
  97
  98static int
  99cxgbit_np_hash_find(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 100{
 101	int stid = -1, bucket = cxgbit_np_hashfn(cnp);
 102	struct np_info *p;
 103
 104	spin_lock(&cdev->np_lock);
 105	for (p = cdev->np_hash_tab[bucket]; p; p = p->next) {
 106		if (p->cnp == cnp) {
 107			stid = p->stid;
 108			break;
 109		}
 110	}
 111	spin_unlock(&cdev->np_lock);
 112
 113	return stid;
 114}
 115
 116static int cxgbit_np_hash_del(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 117{
 118	int stid = -1, bucket = cxgbit_np_hashfn(cnp);
 119	struct np_info *p, **prev = &cdev->np_hash_tab[bucket];
 120
 121	spin_lock(&cdev->np_lock);
 122	for (p = *prev; p; prev = &p->next, p = p->next) {
 123		if (p->cnp == cnp) {
 124			stid = p->stid;
 125			*prev = p->next;
 126			kfree(p);
 127			break;
 128		}
 129	}
 130	spin_unlock(&cdev->np_lock);
 131
 132	return stid;
 133}
 134
 135void _cxgbit_free_cnp(struct kref *kref)
 136{
 137	struct cxgbit_np *cnp;
 138
 139	cnp = container_of(kref, struct cxgbit_np, kref);
 140	kfree(cnp);
 141}
 142
 143static int
 144cxgbit_create_server6(struct cxgbit_device *cdev, unsigned int stid,
 145		      struct cxgbit_np *cnp)
 146{
 147	struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
 148				     &cnp->com.local_addr;
 149	int addr_type;
 150	int ret;
 151
 152	pr_debug("%s: dev = %s; stid = %u; sin6_port = %u\n",
 153		 __func__, cdev->lldi.ports[0]->name, stid, sin6->sin6_port);
 154
 155	addr_type = ipv6_addr_type((const struct in6_addr *)
 156				   &sin6->sin6_addr);
 157	if (addr_type != IPV6_ADDR_ANY) {
 158		ret = cxgb4_clip_get(cdev->lldi.ports[0],
 159				     (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 160		if (ret) {
 161			pr_err("Unable to find clip table entry. laddr %pI6. Error:%d.\n",
 162			       sin6->sin6_addr.s6_addr, ret);
 163			return -ENOMEM;
 164		}
 165	}
 166
 167	cxgbit_get_cnp(cnp);
 168	cxgbit_init_wr_wait(&cnp->com.wr_wait);
 169
 170	ret = cxgb4_create_server6(cdev->lldi.ports[0],
 171				   stid, &sin6->sin6_addr,
 172				   sin6->sin6_port,
 173				   cdev->lldi.rxq_ids[0]);
 174	if (!ret)
 175		ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,
 176					    0, 10, __func__);
 177	else if (ret > 0)
 178		ret = net_xmit_errno(ret);
 179	else
 180		cxgbit_put_cnp(cnp);
 181
 182	if (ret) {
 183		if (ret != -ETIMEDOUT)
 184			cxgb4_clip_release(cdev->lldi.ports[0],
 185				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 186
 187		pr_err("create server6 err %d stid %d laddr %pI6 lport %d\n",
 188		       ret, stid, sin6->sin6_addr.s6_addr,
 189		       ntohs(sin6->sin6_port));
 190	}
 191
 192	return ret;
 193}
 194
 195static int
 196cxgbit_create_server4(struct cxgbit_device *cdev, unsigned int stid,
 197		      struct cxgbit_np *cnp)
 198{
 199	struct sockaddr_in *sin = (struct sockaddr_in *)
 200				   &cnp->com.local_addr;
 201	int ret;
 202
 203	pr_debug("%s: dev = %s; stid = %u; sin_port = %u\n",
 204		 __func__, cdev->lldi.ports[0]->name, stid, sin->sin_port);
 205
 206	cxgbit_get_cnp(cnp);
 207	cxgbit_init_wr_wait(&cnp->com.wr_wait);
 208
 209	ret = cxgb4_create_server(cdev->lldi.ports[0],
 210				  stid, sin->sin_addr.s_addr,
 211				  sin->sin_port, 0,
 212				  cdev->lldi.rxq_ids[0]);
 213	if (!ret)
 214		ret = cxgbit_wait_for_reply(cdev,
 215					    &cnp->com.wr_wait,
 216					    0, 10, __func__);
 217	else if (ret > 0)
 218		ret = net_xmit_errno(ret);
 219	else
 220		cxgbit_put_cnp(cnp);
 221
 222	if (ret)
 223		pr_err("create server failed err %d stid %d laddr %pI4 lport %d\n",
 224		       ret, stid, &sin->sin_addr, ntohs(sin->sin_port));
 225	return ret;
 226}
 227
 228struct cxgbit_device *cxgbit_find_device(struct net_device *ndev, u8 *port_id)
 229{
 230	struct cxgbit_device *cdev;
 231	u8 i;
 232
 233	list_for_each_entry(cdev, &cdev_list_head, list) {
 234		struct cxgb4_lld_info *lldi = &cdev->lldi;
 235
 236		for (i = 0; i < lldi->nports; i++) {
 237			if (lldi->ports[i] == ndev) {
 238				if (port_id)
 239					*port_id = i;
 240				return cdev;
 241			}
 242		}
 243	}
 244
 245	return NULL;
 246}
 247
 248static struct net_device *cxgbit_get_real_dev(struct net_device *ndev)
 249{
 250	if (ndev->priv_flags & IFF_BONDING) {
 251		pr_err("Bond devices are not supported. Interface:%s\n",
 252		       ndev->name);
 253		return NULL;
 254	}
 255
 256	if (is_vlan_dev(ndev))
 257		return vlan_dev_real_dev(ndev);
 258
 259	return ndev;
 260}
 261
 262static struct net_device *cxgbit_ipv4_netdev(__be32 saddr)
 263{
 264	struct net_device *ndev;
 265
 266	ndev = __ip_dev_find(&init_net, saddr, false);
 267	if (!ndev)
 268		return NULL;
 269
 270	return cxgbit_get_real_dev(ndev);
 271}
 272
 273static struct net_device *cxgbit_ipv6_netdev(struct in6_addr *addr6)
 274{
 275	struct net_device *ndev = NULL;
 276	bool found = false;
 277
 278	if (IS_ENABLED(CONFIG_IPV6)) {
 279		for_each_netdev_rcu(&init_net, ndev)
 280			if (ipv6_chk_addr(&init_net, addr6, ndev, 1)) {
 281				found = true;
 282				break;
 283			}
 284	}
 285	if (!found)
 286		return NULL;
 287	return cxgbit_get_real_dev(ndev);
 288}
 289
 290static struct cxgbit_device *cxgbit_find_np_cdev(struct cxgbit_np *cnp)
 291{
 292	struct sockaddr_storage *sockaddr = &cnp->com.local_addr;
 293	int ss_family = sockaddr->ss_family;
 294	struct net_device *ndev = NULL;
 295	struct cxgbit_device *cdev = NULL;
 296
 297	rcu_read_lock();
 298	if (ss_family == AF_INET) {
 299		struct sockaddr_in *sin;
 300
 301		sin = (struct sockaddr_in *)sockaddr;
 302		ndev = cxgbit_ipv4_netdev(sin->sin_addr.s_addr);
 303	} else if (ss_family == AF_INET6) {
 304		struct sockaddr_in6 *sin6;
 305
 306		sin6 = (struct sockaddr_in6 *)sockaddr;
 307		ndev = cxgbit_ipv6_netdev(&sin6->sin6_addr);
 308	}
 309	if (!ndev)
 310		goto out;
 311
 312	cdev = cxgbit_find_device(ndev, NULL);
 313out:
 314	rcu_read_unlock();
 315	return cdev;
 316}
 317
 318static bool cxgbit_inaddr_any(struct cxgbit_np *cnp)
 319{
 320	struct sockaddr_storage *sockaddr = &cnp->com.local_addr;
 321	int ss_family = sockaddr->ss_family;
 322	int addr_type;
 323
 324	if (ss_family == AF_INET) {
 325		struct sockaddr_in *sin;
 326
 327		sin = (struct sockaddr_in *)sockaddr;
 328		if (sin->sin_addr.s_addr == htonl(INADDR_ANY))
 329			return true;
 330	} else if (ss_family == AF_INET6) {
 331		struct sockaddr_in6 *sin6;
 332
 333		sin6 = (struct sockaddr_in6 *)sockaddr;
 334		addr_type = ipv6_addr_type((const struct in6_addr *)
 335				&sin6->sin6_addr);
 336		if (addr_type == IPV6_ADDR_ANY)
 337			return true;
 338	}
 339	return false;
 340}
 341
 342static int
 343__cxgbit_setup_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 344{
 345	int stid, ret;
 346	int ss_family = cnp->com.local_addr.ss_family;
 347
 348	if (!test_bit(CDEV_STATE_UP, &cdev->flags))
 349		return -EINVAL;
 350
 351	stid = cxgb4_alloc_stid(cdev->lldi.tids, ss_family, cnp);
 352	if (stid < 0)
 353		return -EINVAL;
 354
 355	if (!cxgbit_np_hash_add(cdev, cnp, stid)) {
 356		cxgb4_free_stid(cdev->lldi.tids, stid, ss_family);
 357		return -EINVAL;
 358	}
 359
 360	if (ss_family == AF_INET)
 361		ret = cxgbit_create_server4(cdev, stid, cnp);
 362	else
 363		ret = cxgbit_create_server6(cdev, stid, cnp);
 364
 365	if (ret) {
 366		if (ret != -ETIMEDOUT)
 367			cxgb4_free_stid(cdev->lldi.tids, stid,
 368					ss_family);
 369		cxgbit_np_hash_del(cdev, cnp);
 370		return ret;
 371	}
 372	return ret;
 373}
 374
 375static int cxgbit_setup_cdev_np(struct cxgbit_np *cnp)
 376{
 377	struct cxgbit_device *cdev;
 378	int ret = -1;
 379
 380	mutex_lock(&cdev_list_lock);
 381	cdev = cxgbit_find_np_cdev(cnp);
 382	if (!cdev)
 383		goto out;
 384
 385	if (cxgbit_np_hash_find(cdev, cnp) >= 0)
 386		goto out;
 387
 388	if (__cxgbit_setup_cdev_np(cdev, cnp))
 389		goto out;
 390
 391	cnp->com.cdev = cdev;
 392	ret = 0;
 393out:
 394	mutex_unlock(&cdev_list_lock);
 395	return ret;
 396}
 397
 398static int cxgbit_setup_all_np(struct cxgbit_np *cnp)
 399{
 400	struct cxgbit_device *cdev;
 401	int ret;
 402	u32 count = 0;
 403
 404	mutex_lock(&cdev_list_lock);
 405	list_for_each_entry(cdev, &cdev_list_head, list) {
 406		if (cxgbit_np_hash_find(cdev, cnp) >= 0) {
 407			mutex_unlock(&cdev_list_lock);
 408			return -1;
 409		}
 410	}
 411
 412	list_for_each_entry(cdev, &cdev_list_head, list) {
 413		ret = __cxgbit_setup_cdev_np(cdev, cnp);
 414		if (ret == -ETIMEDOUT)
 415			break;
 416		if (ret != 0)
 417			continue;
 418		count++;
 419	}
 420	mutex_unlock(&cdev_list_lock);
 421
 422	return count ? 0 : -1;
 423}
 424
 425int cxgbit_setup_np(struct iscsi_np *np, struct sockaddr_storage *ksockaddr)
 426{
 427	struct cxgbit_np *cnp;
 428	int ret;
 429
 430	if ((ksockaddr->ss_family != AF_INET) &&
 431	    (ksockaddr->ss_family != AF_INET6))
 432		return -EINVAL;
 433
 434	cnp = kzalloc(sizeof(*cnp), GFP_KERNEL);
 435	if (!cnp)
 436		return -ENOMEM;
 437
 438	init_waitqueue_head(&cnp->accept_wait);
 439	init_completion(&cnp->com.wr_wait.completion);
 440	init_completion(&cnp->accept_comp);
 441	INIT_LIST_HEAD(&cnp->np_accept_list);
 442	spin_lock_init(&cnp->np_accept_lock);
 443	kref_init(&cnp->kref);
 444	memcpy(&np->np_sockaddr, ksockaddr,
 445	       sizeof(struct sockaddr_storage));
 446	memcpy(&cnp->com.local_addr, &np->np_sockaddr,
 447	       sizeof(cnp->com.local_addr));
 448
 449	cnp->np = np;
 450	cnp->com.cdev = NULL;
 451
 452	if (cxgbit_inaddr_any(cnp))
 453		ret = cxgbit_setup_all_np(cnp);
 454	else
 455		ret = cxgbit_setup_cdev_np(cnp);
 456
 457	if (ret) {
 458		cxgbit_put_cnp(cnp);
 459		return -EINVAL;
 460	}
 461
 462	np->np_context = cnp;
 463	cnp->com.state = CSK_STATE_LISTEN;
 464	return 0;
 465}
 466
 467static void
 468cxgbit_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn,
 469		     struct cxgbit_sock *csk)
 470{
 471	conn->login_family = np->np_sockaddr.ss_family;
 472	conn->login_sockaddr = csk->com.remote_addr;
 473	conn->local_sockaddr = csk->com.local_addr;
 474}
 475
 476int cxgbit_accept_np(struct iscsi_np *np, struct iscsit_conn *conn)
 477{
 478	struct cxgbit_np *cnp = np->np_context;
 479	struct cxgbit_sock *csk;
 480	int ret = 0;
 481
 482accept_wait:
 483	ret = wait_for_completion_interruptible(&cnp->accept_comp);
 484	if (ret)
 485		return -ENODEV;
 486
 487	spin_lock_bh(&np->np_thread_lock);
 488	if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
 489		spin_unlock_bh(&np->np_thread_lock);
 490		/**
 491		 * No point in stalling here when np_thread
 492		 * is in state RESET/SHUTDOWN/EXIT - bail
 493		 **/
 494		return -ENODEV;
 495	}
 496	spin_unlock_bh(&np->np_thread_lock);
 497
 498	spin_lock_bh(&cnp->np_accept_lock);
 499	if (list_empty(&cnp->np_accept_list)) {
 500		spin_unlock_bh(&cnp->np_accept_lock);
 501		goto accept_wait;
 502	}
 503
 504	csk = list_first_entry(&cnp->np_accept_list,
 505			       struct cxgbit_sock,
 506			       accept_node);
 507
 508	list_del_init(&csk->accept_node);
 509	spin_unlock_bh(&cnp->np_accept_lock);
 510	conn->context = csk;
 511	csk->conn = conn;
 512
 513	cxgbit_set_conn_info(np, conn, csk);
 514	return 0;
 515}
 516
 517static int
 518__cxgbit_free_cdev_np(struct cxgbit_device *cdev, struct cxgbit_np *cnp)
 519{
 520	int stid, ret;
 521	bool ipv6 = false;
 522
 523	stid = cxgbit_np_hash_del(cdev, cnp);
 524	if (stid < 0)
 525		return -EINVAL;
 526	if (!test_bit(CDEV_STATE_UP, &cdev->flags))
 527		return -EINVAL;
 528
 529	if (cnp->np->np_sockaddr.ss_family == AF_INET6)
 530		ipv6 = true;
 531
 532	cxgbit_get_cnp(cnp);
 533	cxgbit_init_wr_wait(&cnp->com.wr_wait);
 534	ret = cxgb4_remove_server(cdev->lldi.ports[0], stid,
 535				  cdev->lldi.rxq_ids[0], ipv6);
 536
 537	if (ret > 0)
 538		ret = net_xmit_errno(ret);
 539
 540	if (ret) {
 541		cxgbit_put_cnp(cnp);
 542		return ret;
 543	}
 544
 545	ret = cxgbit_wait_for_reply(cdev, &cnp->com.wr_wait,
 546				    0, 10, __func__);
 547	if (ret == -ETIMEDOUT)
 548		return ret;
 549
 550	if (ipv6 && cnp->com.cdev) {
 551		struct sockaddr_in6 *sin6;
 552
 553		sin6 = (struct sockaddr_in6 *)&cnp->com.local_addr;
 554		cxgb4_clip_release(cdev->lldi.ports[0],
 555				   (const u32 *)&sin6->sin6_addr.s6_addr,
 556				   1);
 557	}
 558
 559	cxgb4_free_stid(cdev->lldi.tids, stid,
 560			cnp->com.local_addr.ss_family);
 561	return 0;
 562}
 563
 564static void cxgbit_free_all_np(struct cxgbit_np *cnp)
 565{
 566	struct cxgbit_device *cdev;
 567	int ret;
 568
 569	mutex_lock(&cdev_list_lock);
 570	list_for_each_entry(cdev, &cdev_list_head, list) {
 571		ret = __cxgbit_free_cdev_np(cdev, cnp);
 572		if (ret == -ETIMEDOUT)
 573			break;
 574	}
 575	mutex_unlock(&cdev_list_lock);
 576}
 577
 578static void cxgbit_free_cdev_np(struct cxgbit_np *cnp)
 579{
 580	struct cxgbit_device *cdev;
 581	bool found = false;
 582
 583	mutex_lock(&cdev_list_lock);
 584	list_for_each_entry(cdev, &cdev_list_head, list) {
 585		if (cdev == cnp->com.cdev) {
 586			found = true;
 587			break;
 588		}
 589	}
 590	if (!found)
 591		goto out;
 592
 593	__cxgbit_free_cdev_np(cdev, cnp);
 594out:
 595	mutex_unlock(&cdev_list_lock);
 596}
 597
 598static void __cxgbit_free_conn(struct cxgbit_sock *csk);
 599
 600void cxgbit_free_np(struct iscsi_np *np)
 601{
 602	struct cxgbit_np *cnp = np->np_context;
 603	struct cxgbit_sock *csk, *tmp;
 604
 605	cnp->com.state = CSK_STATE_DEAD;
 606	if (cnp->com.cdev)
 607		cxgbit_free_cdev_np(cnp);
 608	else
 609		cxgbit_free_all_np(cnp);
 610
 611	spin_lock_bh(&cnp->np_accept_lock);
 612	list_for_each_entry_safe(csk, tmp, &cnp->np_accept_list, accept_node) {
 613		list_del_init(&csk->accept_node);
 614		__cxgbit_free_conn(csk);
 615	}
 616	spin_unlock_bh(&cnp->np_accept_lock);
 617
 618	np->np_context = NULL;
 619	cxgbit_put_cnp(cnp);
 620}
 621
 622static void cxgbit_send_halfclose(struct cxgbit_sock *csk)
 623{
 624	struct sk_buff *skb;
 625	u32 len = roundup(sizeof(struct cpl_close_con_req), 16);
 626
 627	skb = alloc_skb(len, GFP_ATOMIC);
 628	if (!skb)
 629		return;
 630
 631	cxgb_mk_close_con_req(skb, len, csk->tid, csk->txq_idx,
 632			      NULL, NULL);
 633
 634	cxgbit_skcb_flags(skb) |= SKCBF_TX_FLAG_COMPL;
 635	__skb_queue_tail(&csk->txq, skb);
 636	cxgbit_push_tx_frames(csk);
 637}
 638
 639static void cxgbit_arp_failure_discard(void *handle, struct sk_buff *skb)
 640{
 641	struct cxgbit_sock *csk = handle;
 642
 643	pr_debug("%s cxgbit_device %p\n", __func__, handle);
 644	kfree_skb(skb);
 645	cxgbit_put_csk(csk);
 646}
 647
 648static void cxgbit_abort_arp_failure(void *handle, struct sk_buff *skb)
 649{
 650	struct cxgbit_device *cdev = handle;
 651	struct cpl_abort_req *req = cplhdr(skb);
 652
 653	pr_debug("%s cdev %p\n", __func__, cdev);
 654	req->cmd = CPL_ABORT_NO_RST;
 655	cxgbit_ofld_send(cdev, skb);
 656}
 657
 658static int cxgbit_send_abort_req(struct cxgbit_sock *csk)
 659{
 660	struct sk_buff *skb;
 661	u32 len = roundup(sizeof(struct cpl_abort_req), 16);
 662
 663	pr_debug("%s: csk %p tid %u; state %d\n",
 664		 __func__, csk, csk->tid, csk->com.state);
 665
 666	__skb_queue_purge(&csk->txq);
 667
 668	if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))
 669		cxgbit_send_tx_flowc_wr(csk);
 670
 671	skb = __skb_dequeue(&csk->skbq);
 672	cxgb_mk_abort_req(skb, len, csk->tid, csk->txq_idx,
 673			  csk->com.cdev, cxgbit_abort_arp_failure);
 674
 675	return cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
 676}
 677
 678static void
 679__cxgbit_abort_conn(struct cxgbit_sock *csk, struct sk_buff *skb)
 680{
 681	__kfree_skb(skb);
 682
 683	if (csk->com.state != CSK_STATE_ESTABLISHED)
 684		goto no_abort;
 685
 686	set_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags);
 687	csk->com.state = CSK_STATE_ABORTING;
 688
 689	cxgbit_send_abort_req(csk);
 690
 691	return;
 692
 693no_abort:
 694	cxgbit_wake_up(&csk->com.wr_wait, __func__, CPL_ERR_NONE);
 695	cxgbit_put_csk(csk);
 696}
 697
 698void cxgbit_abort_conn(struct cxgbit_sock *csk)
 699{
 700	struct sk_buff *skb = alloc_skb(0, GFP_KERNEL | __GFP_NOFAIL);
 701
 702	cxgbit_get_csk(csk);
 703	cxgbit_init_wr_wait(&csk->com.wr_wait);
 704
 705	spin_lock_bh(&csk->lock);
 706	if (csk->lock_owner) {
 707		cxgbit_skcb_rx_backlog_fn(skb) = __cxgbit_abort_conn;
 708		__skb_queue_tail(&csk->backlogq, skb);
 709	} else {
 710		__cxgbit_abort_conn(csk, skb);
 711	}
 712	spin_unlock_bh(&csk->lock);
 713
 714	cxgbit_wait_for_reply(csk->com.cdev, &csk->com.wr_wait,
 715			      csk->tid, 600, __func__);
 716}
 717
 718static void __cxgbit_free_conn(struct cxgbit_sock *csk)
 719{
 720	struct iscsit_conn *conn = csk->conn;
 721	bool release = false;
 722
 723	pr_debug("%s: state %d\n",
 724		 __func__, csk->com.state);
 725
 726	spin_lock_bh(&csk->lock);
 727	switch (csk->com.state) {
 728	case CSK_STATE_ESTABLISHED:
 729		if (conn && (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)) {
 730			csk->com.state = CSK_STATE_CLOSING;
 731			cxgbit_send_halfclose(csk);
 732		} else {
 733			csk->com.state = CSK_STATE_ABORTING;
 734			cxgbit_send_abort_req(csk);
 735		}
 736		break;
 737	case CSK_STATE_CLOSING:
 738		csk->com.state = CSK_STATE_MORIBUND;
 739		cxgbit_send_halfclose(csk);
 740		break;
 741	case CSK_STATE_DEAD:
 742		release = true;
 743		break;
 744	default:
 745		pr_err("%s: csk %p; state %d\n",
 746		       __func__, csk, csk->com.state);
 747	}
 748	spin_unlock_bh(&csk->lock);
 749
 750	if (release)
 751		cxgbit_put_csk(csk);
 752}
 753
 754void cxgbit_free_conn(struct iscsit_conn *conn)
 755{
 756	__cxgbit_free_conn(conn->context);
 757}
 758
 759static void cxgbit_set_emss(struct cxgbit_sock *csk, u16 opt)
 760{
 761	csk->emss = csk->com.cdev->lldi.mtus[TCPOPT_MSS_G(opt)] -
 762			((csk->com.remote_addr.ss_family == AF_INET) ?
 763			sizeof(struct iphdr) : sizeof(struct ipv6hdr)) -
 764			sizeof(struct tcphdr);
 765	csk->mss = csk->emss;
 766	if (TCPOPT_TSTAMP_G(opt))
 767		csk->emss -= round_up(TCPOLEN_TIMESTAMP, 4);
 768	if (csk->emss < 128)
 769		csk->emss = 128;
 770	if (csk->emss & 7)
 771		pr_info("Warning: misaligned mtu idx %u mss %u emss=%u\n",
 772			TCPOPT_MSS_G(opt), csk->mss, csk->emss);
 773	pr_debug("%s mss_idx %u mss %u emss=%u\n", __func__, TCPOPT_MSS_G(opt),
 774		 csk->mss, csk->emss);
 775}
 776
 777static void cxgbit_free_skb(struct cxgbit_sock *csk)
 778{
 779	struct sk_buff *skb;
 780
 781	__skb_queue_purge(&csk->txq);
 782	__skb_queue_purge(&csk->rxq);
 783	__skb_queue_purge(&csk->backlogq);
 784	__skb_queue_purge(&csk->ppodq);
 785	__skb_queue_purge(&csk->skbq);
 786
 787	while ((skb = cxgbit_sock_dequeue_wr(csk)))
 788		kfree_skb(skb);
 789
 790	__kfree_skb(csk->lro_hskb);
 791}
 792
 793void _cxgbit_free_csk(struct kref *kref)
 794{
 795	struct cxgbit_sock *csk;
 796	struct cxgbit_device *cdev;
 797
 798	csk = container_of(kref, struct cxgbit_sock, kref);
 799
 800	pr_debug("%s csk %p state %d\n", __func__, csk, csk->com.state);
 801
 802	if (csk->com.local_addr.ss_family == AF_INET6) {
 803		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
 804					     &csk->com.local_addr;
 805		cxgb4_clip_release(csk->com.cdev->lldi.ports[0],
 806				   (const u32 *)
 807				   &sin6->sin6_addr.s6_addr, 1);
 808	}
 809
 810	cxgb4_remove_tid(csk->com.cdev->lldi.tids, 0, csk->tid,
 811			 csk->com.local_addr.ss_family);
 812	dst_release(csk->dst);
 813	cxgb4_l2t_release(csk->l2t);
 814
 815	cdev = csk->com.cdev;
 816	spin_lock_bh(&cdev->cskq.lock);
 817	list_del(&csk->list);
 818	spin_unlock_bh(&cdev->cskq.lock);
 819
 820	cxgbit_free_skb(csk);
 821	cxgbit_put_cnp(csk->cnp);
 822	cxgbit_put_cdev(cdev);
 823
 824	kfree(csk);
 825}
 826
 827static void cxgbit_set_tcp_window(struct cxgbit_sock *csk, struct port_info *pi)
 828{
 829	unsigned int linkspeed;
 830	u8 scale;
 831
 832	linkspeed = pi->link_cfg.speed;
 833	scale = linkspeed / SPEED_10000;
 834
 835#define CXGBIT_10G_RCV_WIN (256 * 1024)
 836	csk->rcv_win = CXGBIT_10G_RCV_WIN;
 837	if (scale)
 838		csk->rcv_win *= scale;
 839	csk->rcv_win = min(csk->rcv_win, RCV_BUFSIZ_M << 10);
 840
 841#define CXGBIT_10G_SND_WIN (256 * 1024)
 842	csk->snd_win = CXGBIT_10G_SND_WIN;
 843	if (scale)
 844		csk->snd_win *= scale;
 845	csk->snd_win = min(csk->snd_win, 512U * 1024);
 846
 847	pr_debug("%s snd_win %d rcv_win %d\n",
 848		 __func__, csk->snd_win, csk->rcv_win);
 849}
 850
 851#ifdef CONFIG_CHELSIO_T4_DCB
 852static u8 cxgbit_get_iscsi_dcb_state(struct net_device *ndev)
 853{
 854	return ndev->dcbnl_ops->getstate(ndev);
 855}
 856
 857static int cxgbit_select_priority(int pri_mask)
 858{
 859	if (!pri_mask)
 860		return 0;
 861
 862	return (ffs(pri_mask) - 1);
 863}
 864
 865static u8 cxgbit_get_iscsi_dcb_priority(struct net_device *ndev, u16 local_port)
 866{
 867	int ret;
 868	u8 caps;
 869
 870	struct dcb_app iscsi_dcb_app = {
 871		.protocol = local_port
 872	};
 873
 874	ret = (int)ndev->dcbnl_ops->getcap(ndev, DCB_CAP_ATTR_DCBX, &caps);
 875
 876	if (ret)
 877		return 0;
 878
 879	if (caps & DCB_CAP_DCBX_VER_IEEE) {
 880		iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_STREAM;
 881		ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
 882		if (!ret) {
 883			iscsi_dcb_app.selector = IEEE_8021QAZ_APP_SEL_ANY;
 884			ret = dcb_ieee_getapp_mask(ndev, &iscsi_dcb_app);
 885		}
 886	} else if (caps & DCB_CAP_DCBX_VER_CEE) {
 887		iscsi_dcb_app.selector = DCB_APP_IDTYPE_PORTNUM;
 888
 889		ret = dcb_getapp(ndev, &iscsi_dcb_app);
 890	}
 891
 892	pr_info("iSCSI priority is set to %u\n", cxgbit_select_priority(ret));
 893
 894	return cxgbit_select_priority(ret);
 895}
 896#endif
 897
 898static int
 899cxgbit_offload_init(struct cxgbit_sock *csk, int iptype, __u8 *peer_ip,
 900		    u16 local_port, struct dst_entry *dst,
 901		    struct cxgbit_device *cdev)
 902{
 903	struct neighbour *n;
 904	int ret, step;
 905	struct net_device *ndev;
 906	u16 rxq_idx, port_id;
 907#ifdef CONFIG_CHELSIO_T4_DCB
 908	u8 priority = 0;
 909#endif
 910
 911	n = dst_neigh_lookup(dst, peer_ip);
 912	if (!n)
 913		return -ENODEV;
 914
 915	rcu_read_lock();
 916	if (!(n->nud_state & NUD_VALID))
 917		neigh_event_send(n, NULL);
 918
 919	ret = -ENOMEM;
 920	if (n->dev->flags & IFF_LOOPBACK) {
 921		if (iptype == 4)
 922			ndev = cxgbit_ipv4_netdev(*(__be32 *)peer_ip);
 923		else if (IS_ENABLED(CONFIG_IPV6))
 924			ndev = cxgbit_ipv6_netdev((struct in6_addr *)peer_ip);
 925		else
 926			ndev = NULL;
 927
 928		if (!ndev) {
 929			ret = -ENODEV;
 930			goto out;
 931		}
 932
 933		csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t,
 934					 n, ndev, 0);
 935		if (!csk->l2t)
 936			goto out;
 937		csk->mtu = ndev->mtu;
 938		csk->tx_chan = cxgb4_port_chan(ndev);
 939		csk->smac_idx =
 940			       ((struct port_info *)netdev_priv(ndev))->smt_idx;
 941		step = cdev->lldi.ntxq /
 942			cdev->lldi.nchan;
 943		csk->txq_idx = cxgb4_port_idx(ndev) * step;
 944		step = cdev->lldi.nrxq /
 945			cdev->lldi.nchan;
 946		csk->ctrlq_idx = cxgb4_port_idx(ndev);
 947		csk->rss_qid = cdev->lldi.rxq_ids[
 948				cxgb4_port_idx(ndev) * step];
 949		csk->port_id = cxgb4_port_idx(ndev);
 950		cxgbit_set_tcp_window(csk,
 951				      (struct port_info *)netdev_priv(ndev));
 952	} else {
 953		ndev = cxgbit_get_real_dev(n->dev);
 954		if (!ndev) {
 955			ret = -ENODEV;
 956			goto out;
 957		}
 958
 959#ifdef CONFIG_CHELSIO_T4_DCB
 960		if (cxgbit_get_iscsi_dcb_state(ndev))
 961			priority = cxgbit_get_iscsi_dcb_priority(ndev,
 962								 local_port);
 963
 964		csk->dcb_priority = priority;
 965
 966		csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, priority);
 967#else
 968		csk->l2t = cxgb4_l2t_get(cdev->lldi.l2t, n, ndev, 0);
 969#endif
 970		if (!csk->l2t)
 971			goto out;
 972		port_id = cxgb4_port_idx(ndev);
 973		csk->mtu = dst_mtu(dst);
 974		csk->tx_chan = cxgb4_port_chan(ndev);
 975		csk->smac_idx =
 976			       ((struct port_info *)netdev_priv(ndev))->smt_idx;
 977		step = cdev->lldi.ntxq /
 978			cdev->lldi.nports;
 979		csk->txq_idx = (port_id * step) +
 980				(cdev->selectq[port_id][0]++ % step);
 981		csk->ctrlq_idx = cxgb4_port_idx(ndev);
 982		step = cdev->lldi.nrxq /
 983			cdev->lldi.nports;
 984		rxq_idx = (port_id * step) +
 985				(cdev->selectq[port_id][1]++ % step);
 986		csk->rss_qid = cdev->lldi.rxq_ids[rxq_idx];
 987		csk->port_id = port_id;
 988		cxgbit_set_tcp_window(csk,
 989				      (struct port_info *)netdev_priv(ndev));
 990	}
 991	ret = 0;
 992out:
 993	rcu_read_unlock();
 994	neigh_release(n);
 995	return ret;
 996}
 997
 998int cxgbit_ofld_send(struct cxgbit_device *cdev, struct sk_buff *skb)
 999{
1000	int ret = 0;
1001
1002	if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
1003		kfree_skb(skb);
1004		pr_err("%s - device not up - dropping\n", __func__);
1005		return -EIO;
1006	}
1007
1008	ret = cxgb4_ofld_send(cdev->lldi.ports[0], skb);
1009	if (ret < 0)
1010		kfree_skb(skb);
1011	return ret < 0 ? ret : 0;
1012}
1013
1014static void cxgbit_release_tid(struct cxgbit_device *cdev, u32 tid)
1015{
1016	u32 len = roundup(sizeof(struct cpl_tid_release), 16);
1017	struct sk_buff *skb;
1018
1019	skb = alloc_skb(len, GFP_ATOMIC);
1020	if (!skb)
1021		return;
1022
1023	cxgb_mk_tid_release(skb, len, tid, 0);
1024	cxgbit_ofld_send(cdev, skb);
1025}
1026
1027int
1028cxgbit_l2t_send(struct cxgbit_device *cdev, struct sk_buff *skb,
1029		struct l2t_entry *l2e)
1030{
1031	int ret = 0;
1032
1033	if (!test_bit(CDEV_STATE_UP, &cdev->flags)) {
1034		kfree_skb(skb);
1035		pr_err("%s - device not up - dropping\n", __func__);
1036		return -EIO;
1037	}
1038
1039	ret = cxgb4_l2t_send(cdev->lldi.ports[0], skb, l2e);
1040	if (ret < 0)
1041		kfree_skb(skb);
1042	return ret < 0 ? ret : 0;
1043}
1044
1045static void cxgbit_send_rx_credits(struct cxgbit_sock *csk, struct sk_buff *skb)
1046{
1047	if (csk->com.state != CSK_STATE_ESTABLISHED) {
1048		__kfree_skb(skb);
1049		return;
1050	}
1051
1052	cxgbit_ofld_send(csk->com.cdev, skb);
1053}
1054
1055/*
1056 * CPL connection rx data ack: host ->
1057 * Send RX credits through an RX_DATA_ACK CPL message.
1058 * Returns the number of credits sent.
1059 */
1060int cxgbit_rx_data_ack(struct cxgbit_sock *csk)
1061{
1062	struct sk_buff *skb;
1063	u32 len = roundup(sizeof(struct cpl_rx_data_ack), 16);
1064	u32 credit_dack;
1065
1066	skb = alloc_skb(len, GFP_KERNEL);
1067	if (!skb)
1068		return -1;
1069
1070	credit_dack = RX_DACK_CHANGE_F | RX_DACK_MODE_V(3) |
1071		      RX_CREDITS_V(csk->rx_credits);
1072
1073	cxgb_mk_rx_data_ack(skb, len, csk->tid, csk->ctrlq_idx,
1074			    credit_dack);
1075
1076	csk->rx_credits = 0;
1077
1078	spin_lock_bh(&csk->lock);
1079	if (csk->lock_owner) {
1080		cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_send_rx_credits;
1081		__skb_queue_tail(&csk->backlogq, skb);
1082		spin_unlock_bh(&csk->lock);
1083		return 0;
1084	}
1085
1086	cxgbit_send_rx_credits(csk, skb);
1087	spin_unlock_bh(&csk->lock);
1088
1089	return 0;
1090}
1091
1092#define FLOWC_WR_NPARAMS_MIN    9
1093#define FLOWC_WR_NPARAMS_MAX	11
1094static int cxgbit_alloc_csk_skb(struct cxgbit_sock *csk)
1095{
1096	struct sk_buff *skb;
1097	u32 len, flowclen;
1098	u8 i;
1099
1100	flowclen = offsetof(struct fw_flowc_wr,
1101			    mnemval[FLOWC_WR_NPARAMS_MAX]);
1102
1103	len = max_t(u32, sizeof(struct cpl_abort_req),
1104		    sizeof(struct cpl_abort_rpl));
1105
1106	len = max(len, flowclen);
1107	len = roundup(len, 16);
1108
1109	for (i = 0; i < 3; i++) {
1110		skb = alloc_skb(len, GFP_ATOMIC);
1111		if (!skb)
1112			goto out;
1113		__skb_queue_tail(&csk->skbq, skb);
1114	}
1115
1116	skb = alloc_skb(LRO_SKB_MIN_HEADROOM, GFP_ATOMIC);
1117	if (!skb)
1118		goto out;
1119
1120	memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1121	csk->lro_hskb = skb;
1122
1123	return 0;
1124out:
1125	__skb_queue_purge(&csk->skbq);
1126	return -ENOMEM;
1127}
1128
1129static void
1130cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req)
1131{
1132	struct sk_buff *skb;
1133	const struct tcphdr *tcph;
1134	struct cpl_t5_pass_accept_rpl *rpl5;
1135	struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1136	unsigned int len = roundup(sizeof(*rpl5), 16);
1137	unsigned int mtu_idx;
1138	u64 opt0;
1139	u32 opt2, hlen;
1140	u32 wscale;
1141	u32 win;
1142
1143	pr_debug("%s csk %p tid %u\n", __func__, csk, csk->tid);
1144
1145	skb = alloc_skb(len, GFP_ATOMIC);
1146	if (!skb) {
1147		cxgbit_put_csk(csk);
1148		return;
1149	}
1150
1151	rpl5 = __skb_put_zero(skb, len);
1152
1153	INIT_TP_WR(rpl5, csk->tid);
1154	OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1155						     csk->tid));
1156	cxgb_best_mtu(csk->com.cdev->lldi.mtus, csk->mtu, &mtu_idx,
1157		      req->tcpopt.tstamp,
1158		      (csk->com.remote_addr.ss_family == AF_INET) ? 0 : 1);
1159	wscale = cxgb_compute_wscale(csk->rcv_win);
1160	/*
1161	 * Specify the largest window that will fit in opt0. The
1162	 * remainder will be specified in the rx_data_ack.
1163	 */
1164	win = csk->rcv_win >> 10;
1165	if (win > RCV_BUFSIZ_M)
1166		win = RCV_BUFSIZ_M;
1167	opt0 =  TCAM_BYPASS_F |
1168		WND_SCALE_V(wscale) |
1169		MSS_IDX_V(mtu_idx) |
1170		L2T_IDX_V(csk->l2t->idx) |
1171		TX_CHAN_V(csk->tx_chan) |
1172		SMAC_SEL_V(csk->smac_idx) |
1173		DSCP_V(csk->tos >> 2) |
1174		ULP_MODE_V(ULP_MODE_ISCSI) |
1175		RCV_BUFSIZ_V(win);
1176
1177	opt2 = RX_CHANNEL_V(0) |
1178		RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1179
1180	if (!is_t5(lldi->adapter_type))
1181		opt2 |= RX_FC_DISABLE_F;
1182
1183	if (req->tcpopt.tstamp)
1184		opt2 |= TSTAMPS_EN_F;
1185	if (req->tcpopt.sack)
1186		opt2 |= SACK_EN_F;
1187	if (wscale)
1188		opt2 |= WND_SCALE_EN_F;
1189
1190	hlen = ntohl(req->hdr_len);
1191
1192	if (is_t5(lldi->adapter_type))
1193		tcph = (struct tcphdr *)((u8 *)(req + 1) +
1194		       ETH_HDR_LEN_G(hlen) + IP_HDR_LEN_G(hlen));
1195	else
1196		tcph = (struct tcphdr *)((u8 *)(req + 1) +
1197		       T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1198
1199	if (tcph->ece && tcph->cwr)
1200		opt2 |= CCTRL_ECN_V(1);
1201
1202	opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1203
1204	opt2 |= T5_ISS_F;
1205	rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
1206
1207	opt2 |= T5_OPT_2_VALID_F;
1208
1209	rpl5->opt0 = cpu_to_be64(opt0);
1210	rpl5->opt2 = cpu_to_be32(opt2);
1211	set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->ctrlq_idx);
1212	t4_set_arp_err_handler(skb, csk, cxgbit_arp_failure_discard);
1213	cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
1214}
1215
1216static void
1217cxgbit_pass_accept_req(struct cxgbit_device *cdev, struct sk_buff *skb)
1218{
1219	struct cxgbit_sock *csk = NULL;
1220	struct cxgbit_np *cnp;
1221	struct cpl_pass_accept_req *req = cplhdr(skb);
1222	unsigned int stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1223	struct tid_info *t = cdev->lldi.tids;
1224	unsigned int tid = GET_TID(req);
1225	u16 peer_mss = ntohs(req->tcpopt.mss);
1226	unsigned short hdrs;
1227
1228	struct dst_entry *dst;
1229	__u8 local_ip[16], peer_ip[16];
1230	__be16 local_port, peer_port;
1231	int ret;
1232	int iptype;
1233
1234	pr_debug("%s: cdev = %p; stid = %u; tid = %u\n",
1235		 __func__, cdev, stid, tid);
1236
1237	cnp = lookup_stid(t, stid);
1238	if (!cnp) {
1239		pr_err("%s connect request on invalid stid %d\n",
1240		       __func__, stid);
1241		goto rel_skb;
1242	}
1243
1244	if (cnp->com.state != CSK_STATE_LISTEN) {
1245		pr_err("%s - listening parent not in CSK_STATE_LISTEN\n",
1246		       __func__);
1247		goto reject;
1248	}
1249
1250	csk = lookup_tid(t, tid);
1251	if (csk) {
1252		pr_err("%s csk not null tid %u\n",
1253		       __func__, tid);
1254		goto rel_skb;
1255	}
1256
1257	cxgb_get_4tuple(req, cdev->lldi.adapter_type, &iptype, local_ip,
1258			peer_ip, &local_port, &peer_port);
1259
1260	/* Find output route */
1261	if (iptype == 4)  {
1262		pr_debug("%s parent sock %p tid %u laddr %pI4 raddr %pI4 "
1263			 "lport %d rport %d peer_mss %d\n"
1264			 , __func__, cnp, tid,
1265			 local_ip, peer_ip, ntohs(local_port),
1266			 ntohs(peer_port), peer_mss);
1267		dst = cxgb_find_route(&cdev->lldi, cxgbit_get_real_dev,
1268				      *(__be32 *)local_ip,
1269				      *(__be32 *)peer_ip,
1270				      local_port, peer_port,
1271				      PASS_OPEN_TOS_G(ntohl(req->tos_stid)));
1272	} else {
1273		pr_debug("%s parent sock %p tid %u laddr %pI6 raddr %pI6 "
1274			 "lport %d rport %d peer_mss %d\n"
1275			 , __func__, cnp, tid,
1276			 local_ip, peer_ip, ntohs(local_port),
1277			 ntohs(peer_port), peer_mss);
1278		dst = cxgb_find_route6(&cdev->lldi, cxgbit_get_real_dev,
1279				       local_ip, peer_ip,
1280				       local_port, peer_port,
1281				       PASS_OPEN_TOS_G(ntohl(req->tos_stid)),
1282				       ((struct sockaddr_in6 *)
1283					&cnp->com.local_addr)->sin6_scope_id);
1284	}
1285	if (!dst) {
1286		pr_err("%s - failed to find dst entry!\n",
1287		       __func__);
1288		goto reject;
1289	}
1290
1291	csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
1292	if (!csk) {
1293		dst_release(dst);
1294		goto rel_skb;
1295	}
1296
1297	ret = cxgbit_offload_init(csk, iptype, peer_ip, ntohs(local_port),
1298				  dst, cdev);
1299	if (ret) {
1300		pr_err("%s - failed to allocate l2t entry!\n",
1301		       __func__);
1302		dst_release(dst);
1303		kfree(csk);
1304		goto reject;
1305	}
1306
1307	kref_init(&csk->kref);
1308	init_completion(&csk->com.wr_wait.completion);
1309
1310	INIT_LIST_HEAD(&csk->accept_node);
1311
1312	hdrs = (iptype == 4 ? sizeof(struct iphdr) : sizeof(struct ipv6hdr)) +
1313		sizeof(struct tcphdr) +	(req->tcpopt.tstamp ? 12 : 0);
1314	if (peer_mss && csk->mtu > (peer_mss + hdrs))
1315		csk->mtu = peer_mss + hdrs;
1316
1317	csk->com.state = CSK_STATE_CONNECTING;
1318	csk->com.cdev = cdev;
1319	csk->cnp = cnp;
1320	csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1321	csk->dst = dst;
1322	csk->tid = tid;
1323	csk->wr_cred = cdev->lldi.wr_cred -
1324			DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);
1325	csk->wr_max_cred = csk->wr_cred;
1326	csk->wr_una_cred = 0;
1327
1328	if (iptype == 4) {
1329		struct sockaddr_in *sin = (struct sockaddr_in *)
1330					  &csk->com.local_addr;
1331		sin->sin_family = AF_INET;
1332		sin->sin_port = local_port;
1333		sin->sin_addr.s_addr = *(__be32 *)local_ip;
1334
1335		sin = (struct sockaddr_in *)&csk->com.remote_addr;
1336		sin->sin_family = AF_INET;
1337		sin->sin_port = peer_port;
1338		sin->sin_addr.s_addr = *(__be32 *)peer_ip;
1339	} else {
1340		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)
1341					    &csk->com.local_addr;
1342
1343		sin6->sin6_family = PF_INET6;
1344		sin6->sin6_port = local_port;
1345		memcpy(sin6->sin6_addr.s6_addr, local_ip, 16);
1346		cxgb4_clip_get(cdev->lldi.ports[0],
1347			       (const u32 *)&sin6->sin6_addr.s6_addr,
1348			       1);
1349
1350		sin6 = (struct sockaddr_in6 *)&csk->com.remote_addr;
1351		sin6->sin6_family = PF_INET6;
1352		sin6->sin6_port = peer_port;
1353		memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16);
1354	}
1355
1356	skb_queue_head_init(&csk->rxq);
1357	skb_queue_head_init(&csk->txq);
1358	skb_queue_head_init(&csk->ppodq);
1359	skb_queue_head_init(&csk->backlogq);
1360	skb_queue_head_init(&csk->skbq);
1361	cxgbit_sock_reset_wr_list(csk);
1362	spin_lock_init(&csk->lock);
1363	init_waitqueue_head(&csk->waitq);
1364	csk->lock_owner = false;
1365
1366	if (cxgbit_alloc_csk_skb(csk)) {
1367		dst_release(dst);
1368		kfree(csk);
1369		goto rel_skb;
1370	}
1371
1372	cxgbit_get_cnp(cnp);
1373	cxgbit_get_cdev(cdev);
1374
1375	spin_lock(&cdev->cskq.lock);
1376	list_add_tail(&csk->list, &cdev->cskq.list);
1377	spin_unlock(&cdev->cskq.lock);
1378	cxgb4_insert_tid(t, csk, tid, csk->com.local_addr.ss_family);
1379	cxgbit_pass_accept_rpl(csk, req);
1380	goto rel_skb;
1381
1382reject:
1383	cxgbit_release_tid(cdev, tid);
1384rel_skb:
1385	__kfree_skb(skb);
1386}
1387
1388static u32
1389cxgbit_tx_flowc_wr_credits(struct cxgbit_sock *csk, u32 *nparamsp,
1390			   u32 *flowclenp)
1391{
1392	u32 nparams, flowclen16, flowclen;
1393
1394	nparams = FLOWC_WR_NPARAMS_MIN;
1395
1396	if (csk->snd_wscale)
1397		nparams++;
1398
1399#ifdef CONFIG_CHELSIO_T4_DCB
1400	nparams++;
1401#endif
1402	flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
1403	flowclen16 = DIV_ROUND_UP(flowclen, 16);
1404	flowclen = flowclen16 * 16;
1405	/*
1406	 * Return the number of 16-byte credits used by the flowc request.
1407	 * Pass back the nparams and actual flowc length if requested.
1408	 */
1409	if (nparamsp)
1410		*nparamsp = nparams;
1411	if (flowclenp)
1412		*flowclenp = flowclen;
1413	return flowclen16;
1414}
1415
1416u32 cxgbit_send_tx_flowc_wr(struct cxgbit_sock *csk)
1417{
1418	struct cxgbit_device *cdev = csk->com.cdev;
1419	struct fw_flowc_wr *flowc;
1420	u32 nparams, flowclen16, flowclen;
1421	struct sk_buff *skb;
1422	u8 index;
1423
1424#ifdef CONFIG_CHELSIO_T4_DCB
1425	u16 vlan = ((struct l2t_entry *)csk->l2t)->vlan;
1426#endif
1427
1428	flowclen16 = cxgbit_tx_flowc_wr_credits(csk, &nparams, &flowclen);
1429
1430	skb = __skb_dequeue(&csk->skbq);
1431	flowc = __skb_put_zero(skb, flowclen);
1432
1433	flowc->op_to_nparams = cpu_to_be32(FW_WR_OP_V(FW_FLOWC_WR) |
1434					   FW_FLOWC_WR_NPARAMS_V(nparams));
1435	flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16_V(flowclen16) |
1436					  FW_WR_FLOWID_V(csk->tid));
1437	flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
1438	flowc->mnemval[0].val = cpu_to_be32(FW_PFVF_CMD_PFN_V
1439					    (csk->com.cdev->lldi.pf));
1440	flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
1441	flowc->mnemval[1].val = cpu_to_be32(csk->tx_chan);
1442	flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
1443	flowc->mnemval[2].val = cpu_to_be32(csk->tx_chan);
1444	flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
1445	flowc->mnemval[3].val = cpu_to_be32(csk->rss_qid);
1446	flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
1447	flowc->mnemval[4].val = cpu_to_be32(csk->snd_nxt);
1448	flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
1449	flowc->mnemval[5].val = cpu_to_be32(csk->rcv_nxt);
1450	flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
1451	flowc->mnemval[6].val = cpu_to_be32(csk->snd_win);
1452	flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
1453	flowc->mnemval[7].val = cpu_to_be32(csk->emss);
1454
1455	flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
1456	if (test_bit(CDEV_ISO_ENABLE, &cdev->flags))
1457		flowc->mnemval[8].val = cpu_to_be32(CXGBIT_MAX_ISO_PAYLOAD);
1458	else
1459		flowc->mnemval[8].val = cpu_to_be32(16384);
1460
1461	index = 9;
1462
1463	if (csk->snd_wscale) {
1464		flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_RCV_SCALE;
1465		flowc->mnemval[index].val = cpu_to_be32(csk->snd_wscale);
1466		index++;
1467	}
1468
1469#ifdef CONFIG_CHELSIO_T4_DCB
1470	flowc->mnemval[index].mnemonic = FW_FLOWC_MNEM_DCBPRIO;
1471	if (vlan == VLAN_NONE) {
1472		pr_warn("csk %u without VLAN Tag on DCB Link\n", csk->tid);
1473		flowc->mnemval[index].val = cpu_to_be32(0);
1474	} else
1475		flowc->mnemval[index].val = cpu_to_be32(
1476				(vlan & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT);
1477#endif
1478
1479	pr_debug("%s: csk %p; tx_chan = %u; rss_qid = %u; snd_seq = %u;"
1480		 " rcv_seq = %u; snd_win = %u; emss = %u\n",
1481		 __func__, csk, csk->tx_chan, csk->rss_qid, csk->snd_nxt,
1482		 csk->rcv_nxt, csk->snd_win, csk->emss);
1483	set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
1484	cxgbit_ofld_send(csk->com.cdev, skb);
1485	return flowclen16;
1486}
1487
1488static int
1489cxgbit_send_tcb_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1490{
1491	spin_lock_bh(&csk->lock);
1492	if (unlikely(csk->com.state != CSK_STATE_ESTABLISHED)) {
1493		spin_unlock_bh(&csk->lock);
1494		pr_err("%s: csk 0x%p, tid %u, state %u\n",
1495		       __func__, csk, csk->tid, csk->com.state);
1496		__kfree_skb(skb);
1497		return -1;
1498	}
1499
1500	cxgbit_get_csk(csk);
1501	cxgbit_init_wr_wait(&csk->com.wr_wait);
1502	cxgbit_ofld_send(csk->com.cdev, skb);
1503	spin_unlock_bh(&csk->lock);
1504
1505	return 0;
1506}
1507
1508int cxgbit_setup_conn_digest(struct cxgbit_sock *csk)
1509{
1510	struct sk_buff *skb;
1511	struct cpl_set_tcb_field *req;
1512	u8 hcrc = csk->submode & CXGBIT_SUBMODE_HCRC;
1513	u8 dcrc = csk->submode & CXGBIT_SUBMODE_DCRC;
1514	unsigned int len = roundup(sizeof(*req), 16);
1515	int ret;
1516
1517	skb = alloc_skb(len, GFP_KERNEL);
1518	if (!skb)
1519		return -ENOMEM;
1520
1521	/*  set up ulp submode */
1522	req = __skb_put_zero(skb, len);
1523
1524	INIT_TP_WR(req, csk->tid);
1525	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1526	req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
1527	req->word_cookie = htons(0);
1528	req->mask = cpu_to_be64(0x3 << 4);
1529	req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
1530				(dcrc ? ULP_CRC_DATA : 0)) << 4);
1531	set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
1532
1533	if (cxgbit_send_tcb_skb(csk, skb))
1534		return -1;
1535
1536	ret = cxgbit_wait_for_reply(csk->com.cdev,
1537				    &csk->com.wr_wait,
1538				    csk->tid, 5, __func__);
1539	if (ret)
1540		return -1;
1541
1542	return 0;
1543}
1544
1545int cxgbit_setup_conn_pgidx(struct cxgbit_sock *csk, u32 pg_idx)
1546{
1547	struct sk_buff *skb;
1548	struct cpl_set_tcb_field *req;
1549	unsigned int len = roundup(sizeof(*req), 16);
1550	int ret;
1551
1552	skb = alloc_skb(len, GFP_KERNEL);
1553	if (!skb)
1554		return -ENOMEM;
1555
1556	req = __skb_put_zero(skb, len);
1557
1558	INIT_TP_WR(req, csk->tid);
1559	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1560	req->reply_ctrl = htons(NO_REPLY_V(0) | QUEUENO_V(csk->rss_qid));
1561	req->word_cookie = htons(0);
1562	req->mask = cpu_to_be64(0x3 << 8);
1563	req->val = cpu_to_be64(pg_idx << 8);
1564	set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->ctrlq_idx);
1565
1566	if (cxgbit_send_tcb_skb(csk, skb))
1567		return -1;
1568
1569	ret = cxgbit_wait_for_reply(csk->com.cdev,
1570				    &csk->com.wr_wait,
1571				    csk->tid, 5, __func__);
1572	if (ret)
1573		return -1;
1574
1575	return 0;
1576}
1577
1578static void
1579cxgbit_pass_open_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1580{
1581	struct cpl_pass_open_rpl *rpl = cplhdr(skb);
1582	struct tid_info *t = cdev->lldi.tids;
1583	unsigned int stid = GET_TID(rpl);
1584	struct cxgbit_np *cnp = lookup_stid(t, stid);
1585
1586	pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1587		 __func__, cnp, stid, rpl->status);
1588
1589	if (!cnp) {
1590		pr_info("%s stid %d lookup failure\n", __func__, stid);
1591		goto rel_skb;
1592	}
1593
1594	cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);
1595	cxgbit_put_cnp(cnp);
1596rel_skb:
1597	__kfree_skb(skb);
1598}
1599
1600static void
1601cxgbit_close_listsrv_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1602{
1603	struct cpl_close_listsvr_rpl *rpl = cplhdr(skb);
1604	struct tid_info *t = cdev->lldi.tids;
1605	unsigned int stid = GET_TID(rpl);
1606	struct cxgbit_np *cnp = lookup_stid(t, stid);
1607
1608	pr_debug("%s: cnp = %p; stid = %u; status = %d\n",
1609		 __func__, cnp, stid, rpl->status);
1610
1611	if (!cnp) {
1612		pr_info("%s stid %d lookup failure\n", __func__, stid);
1613		goto rel_skb;
1614	}
1615
1616	cxgbit_wake_up(&cnp->com.wr_wait, __func__, rpl->status);
1617	cxgbit_put_cnp(cnp);
1618rel_skb:
1619	__kfree_skb(skb);
1620}
1621
1622static void
1623cxgbit_pass_establish(struct cxgbit_device *cdev, struct sk_buff *skb)
1624{
1625	struct cpl_pass_establish *req = cplhdr(skb);
1626	struct tid_info *t = cdev->lldi.tids;
1627	unsigned int tid = GET_TID(req);
1628	struct cxgbit_sock *csk;
1629	struct cxgbit_np *cnp;
1630	u16 tcp_opt = be16_to_cpu(req->tcp_opt);
1631	u32 snd_isn = be32_to_cpu(req->snd_isn);
1632	u32 rcv_isn = be32_to_cpu(req->rcv_isn);
1633
1634	csk = lookup_tid(t, tid);
1635	if (unlikely(!csk)) {
1636		pr_err("can't find connection for tid %u.\n", tid);
1637		goto rel_skb;
1638	}
1639	cnp = csk->cnp;
1640
1641	pr_debug("%s: csk %p; tid %u; cnp %p\n",
1642		 __func__, csk, tid, cnp);
1643
1644	csk->write_seq = snd_isn;
1645	csk->snd_una = snd_isn;
1646	csk->snd_nxt = snd_isn;
1647
1648	csk->rcv_nxt = rcv_isn;
1649
1650	csk->snd_wscale = TCPOPT_SND_WSCALE_G(tcp_opt);
1651	cxgbit_set_emss(csk, tcp_opt);
1652	dst_confirm(csk->dst);
1653	csk->com.state = CSK_STATE_ESTABLISHED;
1654	spin_lock_bh(&cnp->np_accept_lock);
1655	list_add_tail(&csk->accept_node, &cnp->np_accept_list);
1656	spin_unlock_bh(&cnp->np_accept_lock);
1657	complete(&cnp->accept_comp);
1658rel_skb:
1659	__kfree_skb(skb);
1660}
1661
1662static void cxgbit_queue_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1663{
1664	cxgbit_skcb_flags(skb) = 0;
1665	spin_lock_bh(&csk->rxq.lock);
1666	__skb_queue_tail(&csk->rxq, skb);
1667	spin_unlock_bh(&csk->rxq.lock);
1668	wake_up(&csk->waitq);
1669}
1670
1671static void cxgbit_peer_close(struct cxgbit_sock *csk, struct sk_buff *skb)
1672{
1673	pr_debug("%s: csk %p; tid %u; state %d\n",
1674		 __func__, csk, csk->tid, csk->com.state);
1675
1676	switch (csk->com.state) {
1677	case CSK_STATE_ESTABLISHED:
1678		csk->com.state = CSK_STATE_CLOSING;
1679		cxgbit_queue_rx_skb(csk, skb);
1680		return;
1681	case CSK_STATE_CLOSING:
1682		/* simultaneous close */
1683		csk->com.state = CSK_STATE_MORIBUND;
1684		break;
1685	case CSK_STATE_MORIBUND:
1686		csk->com.state = CSK_STATE_DEAD;
1687		cxgbit_put_csk(csk);
1688		break;
1689	case CSK_STATE_ABORTING:
1690		break;
1691	default:
1692		pr_info("%s: cpl_peer_close in bad state %d\n",
1693			__func__, csk->com.state);
1694	}
1695
1696	__kfree_skb(skb);
1697}
1698
1699static void cxgbit_close_con_rpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1700{
1701	pr_debug("%s: csk %p; tid %u; state %d\n",
1702		 __func__, csk, csk->tid, csk->com.state);
1703
1704	switch (csk->com.state) {
1705	case CSK_STATE_CLOSING:
1706		csk->com.state = CSK_STATE_MORIBUND;
1707		break;
1708	case CSK_STATE_MORIBUND:
1709		csk->com.state = CSK_STATE_DEAD;
1710		cxgbit_put_csk(csk);
1711		break;
1712	case CSK_STATE_ABORTING:
1713	case CSK_STATE_DEAD:
1714		break;
1715	default:
1716		pr_info("%s: cpl_close_con_rpl in bad state %d\n",
1717			__func__, csk->com.state);
1718	}
1719
1720	__kfree_skb(skb);
1721}
1722
1723static void cxgbit_abort_req_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
1724{
1725	struct cpl_abort_req_rss *hdr = cplhdr(skb);
1726	unsigned int tid = GET_TID(hdr);
1727	struct sk_buff *rpl_skb;
1728	bool release = false;
1729	bool wakeup_thread = false;
1730	u32 len = roundup(sizeof(struct cpl_abort_rpl), 16);
1731
1732	pr_debug("%s: csk %p; tid %u; state %d\n",
1733		 __func__, csk, tid, csk->com.state);
1734
1735	if (cxgb_is_neg_adv(hdr->status)) {
1736		pr_err("%s: got neg advise %d on tid %u\n",
1737		       __func__, hdr->status, tid);
1738		goto rel_skb;
1739	}
1740
1741	switch (csk->com.state) {
1742	case CSK_STATE_CONNECTING:
1743	case CSK_STATE_MORIBUND:
1744		csk->com.state = CSK_STATE_DEAD;
1745		release = true;
1746		break;
1747	case CSK_STATE_ESTABLISHED:
1748		csk->com.state = CSK_STATE_DEAD;
1749		wakeup_thread = true;
1750		break;
1751	case CSK_STATE_CLOSING:
1752		csk->com.state = CSK_STATE_DEAD;
1753		if (!csk->conn)
1754			release = true;
1755		break;
1756	case CSK_STATE_ABORTING:
1757		break;
1758	default:
1759		pr_info("%s: cpl_abort_req_rss in bad state %d\n",
1760			__func__, csk->com.state);
1761		csk->com.state = CSK_STATE_DEAD;
1762	}
1763
1764	__skb_queue_purge(&csk->txq);
1765
1766	if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags))
1767		cxgbit_send_tx_flowc_wr(csk);
1768
1769	rpl_skb = __skb_dequeue(&csk->skbq);
1770
1771	cxgb_mk_abort_rpl(rpl_skb, len, csk->tid, csk->txq_idx);
1772	cxgbit_ofld_send(csk->com.cdev, rpl_skb);
1773
1774	if (wakeup_thread) {
1775		cxgbit_queue_rx_skb(csk, skb);
1776		return;
1777	}
1778
1779	if (release)
1780		cxgbit_put_csk(csk);
1781rel_skb:
1782	__kfree_skb(skb);
1783}
1784
1785static void cxgbit_abort_rpl_rss(struct cxgbit_sock *csk, struct sk_buff *skb)
1786{
1787	struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1788
1789	pr_debug("%s: csk %p; tid %u; state %d\n",
1790		 __func__, csk, csk->tid, csk->com.state);
1791
1792	switch (csk->com.state) {
1793	case CSK_STATE_ABORTING:
1794		csk->com.state = CSK_STATE_DEAD;
1795		if (test_bit(CSK_ABORT_RPL_WAIT, &csk->com.flags))
1796			cxgbit_wake_up(&csk->com.wr_wait, __func__,
1797				       rpl->status);
1798		cxgbit_put_csk(csk);
1799		break;
1800	default:
1801		pr_info("%s: cpl_abort_rpl_rss in state %d\n",
1802			__func__, csk->com.state);
1803	}
1804
1805	__kfree_skb(skb);
1806}
1807
1808static bool cxgbit_credit_err(const struct cxgbit_sock *csk)
1809{
1810	const struct sk_buff *skb = csk->wr_pending_head;
1811	u32 credit = 0;
1812
1813	if (unlikely(csk->wr_cred > csk->wr_max_cred)) {
1814		pr_err("csk 0x%p, tid %u, credit %u > %u\n",
1815		       csk, csk->tid, csk->wr_cred, csk->wr_max_cred);
1816		return true;
1817	}
1818
1819	while (skb) {
1820		credit += (__force u32)skb->csum;
1821		skb = cxgbit_skcb_tx_wr_next(skb);
1822	}
1823
1824	if (unlikely((csk->wr_cred + credit) != csk->wr_max_cred)) {
1825		pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n",
1826		       csk, csk->tid, csk->wr_cred,
1827		       credit, csk->wr_max_cred);
1828
1829		return true;
1830	}
1831
1832	return false;
1833}
1834
1835static void cxgbit_fw4_ack(struct cxgbit_sock *csk, struct sk_buff *skb)
1836{
1837	struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)cplhdr(skb);
1838	u32 credits = rpl->credits;
1839	u32 snd_una = ntohl(rpl->snd_una);
1840
1841	csk->wr_cred += credits;
1842	if (csk->wr_una_cred > (csk->wr_max_cred - csk->wr_cred))
1843		csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred;
1844
1845	while (credits) {
1846		struct sk_buff *p = cxgbit_sock_peek_wr(csk);
1847		u32 csum;
1848
1849		if (unlikely(!p)) {
1850			pr_err("csk 0x%p,%u, cr %u,%u+%u, empty.\n",
1851			       csk, csk->tid, credits,
1852			       csk->wr_cred, csk->wr_una_cred);
1853			break;
1854		}
1855
1856		csum = (__force u32)p->csum;
1857		if (unlikely(credits < csum)) {
1858			pr_warn("csk 0x%p,%u, cr %u,%u+%u, < %u.\n",
1859				csk,  csk->tid,
1860				credits, csk->wr_cred, csk->wr_una_cred,
1861				csum);
1862			p->csum = (__force __wsum)(csum - credits);
1863			break;
1864		}
1865
1866		cxgbit_sock_dequeue_wr(csk);
1867		credits -= csum;
1868		kfree_skb(p);
1869	}
1870
1871	if (unlikely(cxgbit_credit_err(csk))) {
1872		cxgbit_queue_rx_skb(csk, skb);
1873		return;
1874	}
1875
1876	if (rpl->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
1877		if (unlikely(before(snd_una, csk->snd_una))) {
1878			pr_warn("csk 0x%p,%u, snd_una %u/%u.",
1879				csk, csk->tid, snd_una,
1880				csk->snd_una);
1881			goto rel_skb;
1882		}
1883
1884		if (csk->snd_una != snd_una) {
1885			csk->snd_una = snd_una;
1886			dst_confirm(csk->dst);
1887		}
1888	}
1889
1890	if (skb_queue_len(&csk->txq))
1891		cxgbit_push_tx_frames(csk);
1892
1893rel_skb:
1894	__kfree_skb(skb);
1895}
1896
1897static void cxgbit_set_tcb_rpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1898{
1899	struct cxgbit_sock *csk;
1900	struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
1901	unsigned int tid = GET_TID(rpl);
1902	struct cxgb4_lld_info *lldi = &cdev->lldi;
1903	struct tid_info *t = lldi->tids;
1904
1905	csk = lookup_tid(t, tid);
1906	if (unlikely(!csk)) {
1907		pr_err("can't find connection for tid %u.\n", tid);
1908		goto rel_skb;
1909	} else {
1910		cxgbit_wake_up(&csk->com.wr_wait, __func__, rpl->status);
1911	}
1912
1913	cxgbit_put_csk(csk);
1914rel_skb:
1915	__kfree_skb(skb);
1916}
1917
1918static void cxgbit_rx_data(struct cxgbit_device *cdev, struct sk_buff *skb)
1919{
1920	struct cxgbit_sock *csk;
1921	struct cpl_rx_data *cpl = cplhdr(skb);
1922	unsigned int tid = GET_TID(cpl);
1923	struct cxgb4_lld_info *lldi = &cdev->lldi;
1924	struct tid_info *t = lldi->tids;
1925
1926	csk = lookup_tid(t, tid);
1927	if (unlikely(!csk)) {
1928		pr_err("can't find conn. for tid %u.\n", tid);
1929		goto rel_skb;
1930	}
1931
1932	cxgbit_queue_rx_skb(csk, skb);
1933	return;
1934rel_skb:
1935	__kfree_skb(skb);
1936}
1937
1938static void
1939__cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1940{
1941	spin_lock(&csk->lock);
1942	if (csk->lock_owner) {
1943		__skb_queue_tail(&csk->backlogq, skb);
1944		spin_unlock(&csk->lock);
1945		return;
1946	}
1947
1948	cxgbit_skcb_rx_backlog_fn(skb)(csk, skb);
1949	spin_unlock(&csk->lock);
1950}
1951
1952static void cxgbit_process_rx_cpl(struct cxgbit_sock *csk, struct sk_buff *skb)
1953{
1954	cxgbit_get_csk(csk);
1955	__cxgbit_process_rx_cpl(csk, skb);
1956	cxgbit_put_csk(csk);
1957}
1958
1959static void cxgbit_rx_cpl(struct cxgbit_device *cdev, struct sk_buff *skb)
1960{
1961	struct cxgbit_sock *csk;
1962	struct cpl_tx_data *cpl = cplhdr(skb);
1963	struct cxgb4_lld_info *lldi = &cdev->lldi;
1964	struct tid_info *t = lldi->tids;
1965	unsigned int tid = GET_TID(cpl);
1966	u8 opcode = cxgbit_skcb_rx_opcode(skb);
1967	bool ref = true;
1968
1969	switch (opcode) {
1970	case CPL_FW4_ACK:
1971			cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_fw4_ack;
1972			ref = false;
1973			break;
1974	case CPL_PEER_CLOSE:
1975			cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_peer_close;
1976			break;
1977	case CPL_CLOSE_CON_RPL:
1978			cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_close_con_rpl;
1979			break;
1980	case CPL_ABORT_REQ_RSS:
1981			cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_req_rss;
1982			break;
1983	case CPL_ABORT_RPL_RSS:
1984			cxgbit_skcb_rx_backlog_fn(skb) = cxgbit_abort_rpl_rss;
1985			break;
1986	default:
1987		goto rel_skb;
1988	}
1989
1990	csk = lookup_tid(t, tid);
1991	if (unlikely(!csk)) {
1992		pr_err("can't find conn. for tid %u.\n", tid);
1993		goto rel_skb;
1994	}
1995
1996	if (ref)
1997		cxgbit_process_rx_cpl(csk, skb);
1998	else
1999		__cxgbit_process_rx_cpl(csk, skb);
2000
2001	return;
2002rel_skb:
2003	__kfree_skb(skb);
2004}
2005
2006cxgbit_cplhandler_func cxgbit_cplhandlers[NUM_CPL_CMDS] = {
2007	[CPL_PASS_OPEN_RPL]	= cxgbit_pass_open_rpl,
2008	[CPL_CLOSE_LISTSRV_RPL] = cxgbit_close_listsrv_rpl,
2009	[CPL_PASS_ACCEPT_REQ]	= cxgbit_pass_accept_req,
2010	[CPL_PASS_ESTABLISH]	= cxgbit_pass_establish,
2011	[CPL_SET_TCB_RPL]	= cxgbit_set_tcb_rpl,
2012	[CPL_RX_DATA]		= cxgbit_rx_data,
2013	[CPL_FW4_ACK]		= cxgbit_rx_cpl,
2014	[CPL_PEER_CLOSE]	= cxgbit_rx_cpl,
2015	[CPL_CLOSE_CON_RPL]	= cxgbit_rx_cpl,
2016	[CPL_ABORT_REQ_RSS]	= cxgbit_rx_cpl,
2017	[CPL_ABORT_RPL_RSS]	= cxgbit_rx_cpl,
2018};