chtls_cm.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2018 Chelsio Communications, Inc.
   4 *
   5 * Written by: Atul Gupta (atul.gupta@chelsio.com)
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/list.h>
  10#include <linux/workqueue.h>
  11#include <linux/skbuff.h>
  12#include <linux/timer.h>
  13#include <linux/notifier.h>
  14#include <linux/inetdevice.h>
  15#include <linux/ip.h>
  16#include <linux/tcp.h>
  17#include <linux/sched/signal.h>
  18#include <linux/kallsyms.h>
  19#include <linux/kprobes.h>
  20#include <linux/if_vlan.h>
  21#include <linux/ipv6.h>
  22#include <net/ipv6.h>
  23#include <net/transp_v6.h>
  24#include <net/ip6_route.h>
  25#include <net/inet_common.h>
  26#include <net/tcp.h>
  27#include <net/dst.h>
  28#include <net/tls.h>
  29#include <net/addrconf.h>
  30#include <net/secure_seq.h>
  31
  32#include "chtls.h"
  33#include "chtls_cm.h"
  34#include "clip_tbl.h"
  35#include "t4_tcb.h"
  36
  37/*
  38 * State transitions and actions for close.  Note that if we are in SYN_SENT
  39 * we remain in that state as we cannot control a connection while it's in
  40 * SYN_SENT; such connections are allowed to establish and are then aborted.
  41 */
  42static unsigned char new_state[16] = {
  43	/* current state:     new state:      action: */
  44	/* (Invalid)       */ TCP_CLOSE,
  45	/* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  46	/* TCP_SYN_SENT    */ TCP_SYN_SENT,
  47	/* TCP_SYN_RECV    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  48	/* TCP_FIN_WAIT1   */ TCP_FIN_WAIT1,
  49	/* TCP_FIN_WAIT2   */ TCP_FIN_WAIT2,
  50	/* TCP_TIME_WAIT   */ TCP_CLOSE,
  51	/* TCP_CLOSE       */ TCP_CLOSE,
  52	/* TCP_CLOSE_WAIT  */ TCP_LAST_ACK | TCP_ACTION_FIN,
  53	/* TCP_LAST_ACK    */ TCP_LAST_ACK,
  54	/* TCP_LISTEN      */ TCP_CLOSE,
  55	/* TCP_CLOSING     */ TCP_CLOSING,
  56};
  57
  58static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
  59{
  60	struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
  61
  62	if (!csk)
  63		return NULL;
  64
  65	csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
  66	if (!csk->txdata_skb_cache) {
  67		kfree(csk);
  68		return NULL;
  69	}
  70
  71	kref_init(&csk->kref);
  72	csk->cdev = cdev;
  73	skb_queue_head_init(&csk->txq);
  74	csk->wr_skb_head = NULL;
  75	csk->wr_skb_tail = NULL;
  76	csk->mss = MAX_MSS;
  77	csk->tlshws.ofld = 1;
  78	csk->tlshws.txkey = -1;
  79	csk->tlshws.rxkey = -1;
  80	csk->tlshws.mfs = TLS_MFS;
  81	skb_queue_head_init(&csk->tlshws.sk_recv_queue);
  82	return csk;
  83}
  84
  85static void chtls_sock_release(struct kref *ref)
  86{
  87	struct chtls_sock *csk =
  88		container_of(ref, struct chtls_sock, kref);
  89
  90	kfree(csk);
  91}
  92
  93static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
  94					    struct sock *sk)
  95{
  96	struct adapter *adap = pci_get_drvdata(cdev->pdev);
  97	struct net_device *ndev = cdev->ports[0];
  98#if IS_ENABLED(CONFIG_IPV6)
  99	struct net_device *temp;
 100	int addr_type;
 101#endif
 102	int i;
 103
 104	switch (sk->sk_family) {
 105	case PF_INET:
 106		if (likely(!inet_sk(sk)->inet_rcv_saddr))
 107			return ndev;
 108		ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
 109		break;
 110#if IS_ENABLED(CONFIG_IPV6)
 111	case PF_INET6:
 112		addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
 113		if (likely(addr_type == IPV6_ADDR_ANY))
 114			return ndev;
 115
 116		for_each_netdev_rcu(&init_net, temp) {
 117			if (ipv6_chk_addr(&init_net, (struct in6_addr *)
 118					  &sk->sk_v6_rcv_saddr, temp, 1)) {
 119				ndev = temp;
 120				break;
 121			}
 122		}
 123	break;
 124#endif
 125	default:
 126		return NULL;
 127	}
 128
 129	if (!ndev)
 130		return NULL;
 131
 132	if (is_vlan_dev(ndev))
 133		ndev = vlan_dev_real_dev(ndev);
 134
 135	for_each_port(adap, i)
 136		if (cdev->ports[i] == ndev)
 137			return ndev;
 138	return NULL;
 139}
 140
 141static void assign_rxopt(struct sock *sk, unsigned int opt)
 142{
 143	const struct chtls_dev *cdev;
 144	struct chtls_sock *csk;
 145	struct tcp_sock *tp;
 146
 147	csk = rcu_dereference_sk_user_data(sk);
 148	tp = tcp_sk(sk);
 149
 150	cdev = csk->cdev;
 151	tp->tcp_header_len           = sizeof(struct tcphdr);
 152	tp->rx_opt.mss_clamp         = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
 153	tp->mss_cache                = tp->rx_opt.mss_clamp;
 154	tp->rx_opt.tstamp_ok         = TCPOPT_TSTAMP_G(opt);
 155	tp->rx_opt.snd_wscale        = TCPOPT_SACK_G(opt);
 156	tp->rx_opt.wscale_ok         = TCPOPT_WSCALE_OK_G(opt);
 157	SND_WSCALE(tp)               = TCPOPT_SND_WSCALE_G(opt);
 158	if (!tp->rx_opt.wscale_ok)
 159		tp->rx_opt.rcv_wscale = 0;
 160	if (tp->rx_opt.tstamp_ok) {
 161		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
 162		tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
 163	} else if (csk->opt2 & TSTAMPS_EN_F) {
 164		csk->opt2 &= ~TSTAMPS_EN_F;
 165		csk->mtu_idx = TCPOPT_MSS_G(opt);
 166	}
 167}
 168
 169static void chtls_purge_receive_queue(struct sock *sk)
 170{
 171	struct sk_buff *skb;
 172
 173	while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
 174		skb_dstref_steal(skb);
 175		kfree_skb(skb);
 176	}
 177}
 178
 179static void chtls_purge_write_queue(struct sock *sk)
 180{
 181	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 182	struct sk_buff *skb;
 183
 184	while ((skb = __skb_dequeue(&csk->txq))) {
 185		sk->sk_wmem_queued -= skb->truesize;
 186		__kfree_skb(skb);
 187	}
 188}
 189
 190static void chtls_purge_recv_queue(struct sock *sk)
 191{
 192	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 193	struct chtls_hws *tlsk = &csk->tlshws;
 194	struct sk_buff *skb;
 195
 196	while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
 197		skb_dstref_steal(skb);
 198		kfree_skb(skb);
 199	}
 200}
 201
 202static void abort_arp_failure(void *handle, struct sk_buff *skb)
 203{
 204	struct cpl_abort_req *req = cplhdr(skb);
 205	struct chtls_dev *cdev;
 206
 207	cdev = (struct chtls_dev *)handle;
 208	req->cmd = CPL_ABORT_NO_RST;
 209	cxgb4_ofld_send(cdev->lldi->ports[0], skb);
 210}
 211
 212static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
 213{
 214	if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
 215		__skb_trim(skb, 0);
 216		refcount_inc(&skb->users);
 217	} else {
 218		skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 219	}
 220	return skb;
 221}
 222
 223static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
 224{
 225	struct cpl_abort_req *req;
 226	struct chtls_sock *csk;
 227	struct tcp_sock *tp;
 228
 229	csk = rcu_dereference_sk_user_data(sk);
 230	tp = tcp_sk(sk);
 231
 232	if (!skb)
 233		skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
 234
 235	req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
 236	INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
 237	skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
 238	req->rsvd0 = htonl(tp->snd_nxt);
 239	req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
 240	req->cmd = mode;
 241	t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
 242	send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
 243}
 244
 245static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
 246{
 247	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 248
 249	if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
 250		     !csk->cdev)) {
 251		if (sk->sk_state == TCP_SYN_RECV)
 252			csk_set_flag(csk, CSK_RST_ABORTED);
 253		goto out;
 254	}
 255
 256	if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
 257		struct tcp_sock *tp = tcp_sk(sk);
 258
 259		if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
 260			WARN_ONCE(1, "send tx flowc error");
 261		csk_set_flag(csk, CSK_TX_DATA_SENT);
 262	}
 263
 264	csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
 265	chtls_purge_write_queue(sk);
 266
 267	csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
 268	if (sk->sk_state != TCP_SYN_RECV)
 269		chtls_send_abort(sk, mode, skb);
 270	else
 271		chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W,
 272					    TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0,
 273					    TCB_FIELD_COOKIE_TFLAG, 1);
 274
 275	return;
 276out:
 277	kfree_skb(skb);
 278}
 279
 280static void release_tcp_port(struct sock *sk)
 281{
 282	if (inet_csk(sk)->icsk_bind_hash)
 283		inet_put_port(sk);
 284}
 285
 286static void tcp_uncork(struct sock *sk)
 287{
 288	struct tcp_sock *tp = tcp_sk(sk);
 289
 290	if (tp->nonagle & TCP_NAGLE_CORK) {
 291		tp->nonagle &= ~TCP_NAGLE_CORK;
 292		chtls_tcp_push(sk, 0);
 293	}
 294}
 295
 296static void chtls_close_conn(struct sock *sk)
 297{
 298	struct cpl_close_con_req *req;
 299	struct chtls_sock *csk;
 300	struct sk_buff *skb;
 301	unsigned int tid;
 302	unsigned int len;
 303
 304	len = roundup(sizeof(struct cpl_close_con_req), 16);
 305	csk = rcu_dereference_sk_user_data(sk);
 306	tid = csk->tid;
 307
 308	skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 309	req = (struct cpl_close_con_req *)__skb_put(skb, len);
 310	memset(req, 0, len);
 311	req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
 312			      FW_WR_IMMDLEN_V(sizeof(*req) -
 313					      sizeof(req->wr)));
 314	req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
 315			       FW_WR_FLOWID_V(tid));
 316
 317	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 318
 319	tcp_uncork(sk);
 320	skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
 321	if (sk->sk_state != TCP_SYN_SENT)
 322		chtls_push_frames(csk, 1);
 323}
 324
 325/*
 326 * Perform a state transition during close and return the actions indicated
 327 * for the transition.  Do not make this function inline, the main reason
 328 * it exists at all is to avoid multiple inlining of tcp_set_state.
 329 */
 330static int make_close_transition(struct sock *sk)
 331{
 332	int next = (int)new_state[sk->sk_state];
 333
 334	tcp_set_state(sk, next & TCP_STATE_MASK);
 335	return next & TCP_ACTION_FIN;
 336}
 337
 338void chtls_close(struct sock *sk, long timeout)
 339{
 340	int data_lost, prev_state;
 341	struct chtls_sock *csk;
 342
 343	csk = rcu_dereference_sk_user_data(sk);
 344
 345	lock_sock(sk);
 346	sk->sk_shutdown |= SHUTDOWN_MASK;
 347
 348	data_lost = skb_queue_len(&sk->sk_receive_queue);
 349	data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
 350	chtls_purge_recv_queue(sk);
 351	chtls_purge_receive_queue(sk);
 352
 353	if (sk->sk_state == TCP_CLOSE) {
 354		goto wait;
 355	} else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
 356		chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 357		release_tcp_port(sk);
 358		goto unlock;
 359	} else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
 360		sk->sk_prot->disconnect(sk, 0);
 361	} else if (make_close_transition(sk)) {
 362		chtls_close_conn(sk);
 363	}
 364wait:
 365	if (timeout)
 366		sk_stream_wait_close(sk, timeout);
 367
 368unlock:
 369	prev_state = sk->sk_state;
 370	sock_hold(sk);
 371	sock_orphan(sk);
 372
 373	release_sock(sk);
 374
 375	local_bh_disable();
 376	bh_lock_sock(sk);
 377
 378	if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
 379		goto out;
 380
 381	if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
 382	    !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
 383		struct sk_buff *skb;
 384
 385		skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
 386		if (skb)
 387			chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
 388	}
 389
 390	if (sk->sk_state == TCP_CLOSE)
 391		inet_csk_destroy_sock(sk);
 392
 393out:
 394	bh_unlock_sock(sk);
 395	local_bh_enable();
 396	sock_put(sk);
 397}
 398
 399/*
 400 * Wait until a socket enters on of the given states.
 401 */
 402static int wait_for_states(struct sock *sk, unsigned int states)
 403{
 404	DECLARE_WAITQUEUE(wait, current);
 405	struct socket_wq _sk_wq;
 406	long current_timeo;
 407	int err = 0;
 408
 409	current_timeo = 200;
 410
 411	/*
 412	 * We want this to work even when there's no associated struct socket.
 413	 * In that case we provide a temporary wait_queue_head_t.
 414	 */
 415	if (!sk->sk_wq) {
 416		init_waitqueue_head(&_sk_wq.wait);
 417		_sk_wq.fasync_list = NULL;
 418		init_rcu_head_on_stack(&_sk_wq.rcu);
 419		RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
 420	}
 421
 422	add_wait_queue(sk_sleep(sk), &wait);
 423	while (!sk_in_state(sk, states)) {
 424		if (!current_timeo) {
 425			err = -EBUSY;
 426			break;
 427		}
 428		if (signal_pending(current)) {
 429			err = sock_intr_errno(current_timeo);
 430			break;
 431		}
 432		set_current_state(TASK_UNINTERRUPTIBLE);
 433		release_sock(sk);
 434		if (!sk_in_state(sk, states))
 435			current_timeo = schedule_timeout(current_timeo);
 436		__set_current_state(TASK_RUNNING);
 437		lock_sock(sk);
 438	}
 439	remove_wait_queue(sk_sleep(sk), &wait);
 440
 441	if (rcu_dereference(sk->sk_wq) == &_sk_wq)
 442		sk->sk_wq = NULL;
 443	return err;
 444}
 445
 446int chtls_disconnect(struct sock *sk, int flags)
 447{
 448	struct tcp_sock *tp;
 449	int err;
 450
 451	tp = tcp_sk(sk);
 452	chtls_purge_recv_queue(sk);
 453	chtls_purge_receive_queue(sk);
 454	chtls_purge_write_queue(sk);
 455
 456	if (sk->sk_state != TCP_CLOSE) {
 457		sk->sk_err = ECONNRESET;
 458		chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 459		err = wait_for_states(sk, TCPF_CLOSE);
 460		if (err)
 461			return err;
 462	}
 463	chtls_purge_recv_queue(sk);
 464	chtls_purge_receive_queue(sk);
 465	tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
 466	return tcp_disconnect(sk, flags);
 467}
 468
 469#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
 470				 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
 471void chtls_shutdown(struct sock *sk, int how)
 472{
 473	if ((how & SEND_SHUTDOWN) &&
 474	    sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
 475	    make_close_transition(sk))
 476		chtls_close_conn(sk);
 477}
 478
 479void chtls_destroy_sock(struct sock *sk)
 480{
 481	struct chtls_sock *csk;
 482
 483	csk = rcu_dereference_sk_user_data(sk);
 484	chtls_purge_recv_queue(sk);
 485	csk->ulp_mode = ULP_MODE_NONE;
 486	chtls_purge_write_queue(sk);
 487	free_tls_keyid(sk);
 488	kref_put(&csk->kref, chtls_sock_release);
 489	if (sk->sk_family == AF_INET)
 490		sk->sk_prot = &tcp_prot;
 491#if IS_ENABLED(CONFIG_IPV6)
 492	else
 493		sk->sk_prot = &tcpv6_prot;
 494#endif
 495	sk->sk_prot->destroy(sk);
 496}
 497
 498static void reset_listen_child(struct sock *child)
 499{
 500	struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
 501	struct sk_buff *skb;
 502
 503	skb = alloc_ctrl_skb(csk->txdata_skb_cache,
 504			     sizeof(struct cpl_abort_req));
 505
 506	chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
 507	sock_orphan(child);
 508	tcp_orphan_count_inc();
 509	if (child->sk_state == TCP_CLOSE)
 510		inet_csk_destroy_sock(child);
 511}
 512
 513static void chtls_disconnect_acceptq(struct sock *listen_sk)
 514{
 515	struct request_sock **pprev;
 516
 517	pprev = ACCEPT_QUEUE(listen_sk);
 518	while (*pprev) {
 519		struct request_sock *req = *pprev;
 520
 521		if (req->rsk_ops == &chtls_rsk_ops ||
 522		    req->rsk_ops == &chtls_rsk_opsv6) {
 523			struct sock *child = req->sk;
 524
 525			*pprev = req->dl_next;
 526			sk_acceptq_removed(listen_sk);
 527			reqsk_put(req);
 528			sock_hold(child);
 529			local_bh_disable();
 530			bh_lock_sock(child);
 531			release_tcp_port(child);
 532			reset_listen_child(child);
 533			bh_unlock_sock(child);
 534			local_bh_enable();
 535			sock_put(child);
 536		} else {
 537			pprev = &req->dl_next;
 538		}
 539	}
 540}
 541
 542static int listen_hashfn(const struct sock *sk)
 543{
 544	return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
 545}
 546
 547static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
 548					   struct sock *sk,
 549					   unsigned int stid)
 550{
 551	struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
 552
 553	if (p) {
 554		int key = listen_hashfn(sk);
 555
 556		p->sk = sk;
 557		p->stid = stid;
 558		spin_lock(&cdev->listen_lock);
 559		p->next = cdev->listen_hash_tab[key];
 560		cdev->listen_hash_tab[key] = p;
 561		spin_unlock(&cdev->listen_lock);
 562	}
 563	return p;
 564}
 565
 566static int listen_hash_find(struct chtls_dev *cdev,
 567			    struct sock *sk)
 568{
 569	struct listen_info *p;
 570	int stid = -1;
 571	int key;
 572
 573	key = listen_hashfn(sk);
 574
 575	spin_lock(&cdev->listen_lock);
 576	for (p = cdev->listen_hash_tab[key]; p; p = p->next)
 577		if (p->sk == sk) {
 578			stid = p->stid;
 579			break;
 580		}
 581	spin_unlock(&cdev->listen_lock);
 582	return stid;
 583}
 584
 585static int listen_hash_del(struct chtls_dev *cdev,
 586			   struct sock *sk)
 587{
 588	struct listen_info *p, **prev;
 589	int stid = -1;
 590	int key;
 591
 592	key = listen_hashfn(sk);
 593	prev = &cdev->listen_hash_tab[key];
 594
 595	spin_lock(&cdev->listen_lock);
 596	for (p = *prev; p; prev = &p->next, p = p->next)
 597		if (p->sk == sk) {
 598			stid = p->stid;
 599			*prev = p->next;
 600			kfree(p);
 601			break;
 602		}
 603	spin_unlock(&cdev->listen_lock);
 604	return stid;
 605}
 606
 607static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
 608{
 609	struct request_sock *req;
 610	struct chtls_sock *csk;
 611
 612	csk = rcu_dereference_sk_user_data(child);
 613	req = csk->passive_reap_next;
 614
 615	reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
 616	__skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
 617	chtls_reqsk_free(req);
 618	csk->passive_reap_next = NULL;
 619}
 620
 621static void chtls_reset_synq(struct listen_ctx *listen_ctx)
 622{
 623	struct sock *listen_sk = listen_ctx->lsk;
 624
 625	while (!skb_queue_empty(&listen_ctx->synq)) {
 626		struct chtls_sock *csk =
 627			container_of((struct synq *)skb_peek
 628				(&listen_ctx->synq), struct chtls_sock, synq);
 629		struct sock *child = csk->sk;
 630
 631		cleanup_syn_rcv_conn(child, listen_sk);
 632		sock_hold(child);
 633		local_bh_disable();
 634		bh_lock_sock(child);
 635		release_tcp_port(child);
 636		reset_listen_child(child);
 637		bh_unlock_sock(child);
 638		local_bh_enable();
 639		sock_put(child);
 640	}
 641}
 642
 643int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 644{
 645	struct net_device *ndev;
 646#if IS_ENABLED(CONFIG_IPV6)
 647	bool clip_valid = false;
 648#endif
 649	struct listen_ctx *ctx;
 650	struct adapter *adap;
 651	struct port_info *pi;
 652	int ret = 0;
 653	int stid;
 654
 655	rcu_read_lock();
 656	ndev = chtls_find_netdev(cdev, sk);
 657	rcu_read_unlock();
 658	if (!ndev)
 659		return -EBADF;
 660
 661	pi = netdev_priv(ndev);
 662	adap = pi->adapter;
 663	if (!(adap->flags & CXGB4_FULL_INIT_DONE))
 664		return -EBADF;
 665
 666	if (listen_hash_find(cdev, sk) >= 0)   /* already have it */
 667		return -EADDRINUSE;
 668
 669	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 670	if (!ctx)
 671		return -ENOMEM;
 672
 673	__module_get(THIS_MODULE);
 674	ctx->lsk = sk;
 675	ctx->cdev = cdev;
 676	ctx->state = T4_LISTEN_START_PENDING;
 677	skb_queue_head_init(&ctx->synq);
 678
 679	stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
 680	if (stid < 0)
 681		goto free_ctx;
 682
 683	sock_hold(sk);
 684	if (!listen_hash_add(cdev, sk, stid))
 685		goto free_stid;
 686
 687	if (sk->sk_family == PF_INET) {
 688		ret = cxgb4_create_server(ndev, stid,
 689					  inet_sk(sk)->inet_rcv_saddr,
 690					  inet_sk(sk)->inet_sport, 0,
 691					  cdev->lldi->rxq_ids[0]);
 692#if IS_ENABLED(CONFIG_IPV6)
 693	} else {
 694		int addr_type;
 695
 696		addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
 697		if (addr_type != IPV6_ADDR_ANY) {
 698			ret = cxgb4_clip_get(ndev, (const u32 *)
 699					     &sk->sk_v6_rcv_saddr, 1);
 700			if (ret)
 701				goto del_hash;
 702			clip_valid = true;
 703		}
 704		ret = cxgb4_create_server6(ndev, stid,
 705					   &sk->sk_v6_rcv_saddr,
 706					   inet_sk(sk)->inet_sport,
 707					   cdev->lldi->rxq_ids[0]);
 708#endif
 709	}
 710	if (ret > 0)
 711		ret = net_xmit_errno(ret);
 712	if (ret)
 713		goto del_hash;
 714	return 0;
 715del_hash:
 716#if IS_ENABLED(CONFIG_IPV6)
 717	if (clip_valid)
 718		cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
 719#endif
 720	listen_hash_del(cdev, sk);
 721free_stid:
 722	cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
 723	sock_put(sk);
 724free_ctx:
 725	kfree(ctx);
 726	module_put(THIS_MODULE);
 727	return -EBADF;
 728}
 729
 730void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 731{
 732	struct listen_ctx *listen_ctx;
 733	int stid;
 734
 735	stid = listen_hash_del(cdev, sk);
 736	if (stid < 0)
 737		return;
 738
 739	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 740	chtls_reset_synq(listen_ctx);
 741
 742	cxgb4_remove_server(cdev->lldi->ports[0], stid,
 743			    cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
 744
 745#if IS_ENABLED(CONFIG_IPV6)
 746	if (sk->sk_family == PF_INET6) {
 747		struct net_device *ndev = chtls_find_netdev(cdev, sk);
 748		int addr_type = 0;
 749
 750		addr_type = ipv6_addr_type((const struct in6_addr *)
 751					  &sk->sk_v6_rcv_saddr);
 752		if (addr_type != IPV6_ADDR_ANY)
 753			cxgb4_clip_release(ndev, (const u32 *)
 754					   &sk->sk_v6_rcv_saddr, 1);
 755	}
 756#endif
 757	chtls_disconnect_acceptq(sk);
 758}
 759
 760static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 761{
 762	struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
 763	unsigned int stid = GET_TID(rpl);
 764	struct listen_ctx *listen_ctx;
 765
 766	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 767	if (!listen_ctx)
 768		return CPL_RET_BUF_DONE;
 769
 770	if (listen_ctx->state == T4_LISTEN_START_PENDING) {
 771		listen_ctx->state = T4_LISTEN_STARTED;
 772		return CPL_RET_BUF_DONE;
 773	}
 774
 775	if (rpl->status != CPL_ERR_NONE) {
 776		pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
 777			rpl->status, stid);
 778	} else {
 779		cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 780		sock_put(listen_ctx->lsk);
 781		kfree(listen_ctx);
 782		module_put(THIS_MODULE);
 783	}
 784	return CPL_RET_BUF_DONE;
 785}
 786
 787static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 788{
 789	struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
 790	struct listen_ctx *listen_ctx;
 791	unsigned int stid;
 792	void *data;
 793
 794	stid = GET_TID(rpl);
 795	data = lookup_stid(cdev->tids, stid);
 796	listen_ctx = (struct listen_ctx *)data;
 797
 798	if (rpl->status != CPL_ERR_NONE) {
 799		pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
 800			rpl->status, stid);
 801	} else {
 802		cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 803		sock_put(listen_ctx->lsk);
 804		kfree(listen_ctx);
 805		module_put(THIS_MODULE);
 806	}
 807	return CPL_RET_BUF_DONE;
 808}
 809
 810static void chtls_purge_wr_queue(struct sock *sk)
 811{
 812	struct sk_buff *skb;
 813
 814	while ((skb = dequeue_wr(sk)) != NULL)
 815		kfree_skb(skb);
 816}
 817
 818static void chtls_release_resources(struct sock *sk)
 819{
 820	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 821	struct chtls_dev *cdev = csk->cdev;
 822	unsigned int tid = csk->tid;
 823	struct tid_info *tids;
 824
 825	if (!cdev)
 826		return;
 827
 828	tids = cdev->tids;
 829	kfree_skb(csk->txdata_skb_cache);
 830	csk->txdata_skb_cache = NULL;
 831
 832	if (csk->wr_credits != csk->wr_max_credits) {
 833		chtls_purge_wr_queue(sk);
 834		chtls_reset_wr_list(csk);
 835	}
 836
 837	if (csk->l2t_entry) {
 838		cxgb4_l2t_release(csk->l2t_entry);
 839		csk->l2t_entry = NULL;
 840	}
 841
 842	if (sk->sk_state != TCP_SYN_SENT) {
 843		cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
 844		sock_put(sk);
 845	}
 846}
 847
 848static void chtls_conn_done(struct sock *sk)
 849{
 850	if (sock_flag(sk, SOCK_DEAD))
 851		chtls_purge_receive_queue(sk);
 852	sk_wakeup_sleepers(sk, 0);
 853	tcp_done(sk);
 854}
 855
 856static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
 857{
 858	/*
 859	 * If the server is still open we clean up the child connection,
 860	 * otherwise the server already did the clean up as it was purging
 861	 * its SYN queue and the skb was just sitting in its backlog.
 862	 */
 863	if (likely(parent->sk_state == TCP_LISTEN)) {
 864		cleanup_syn_rcv_conn(child, parent);
 865		/* Without the below call to sock_orphan,
 866		 * we leak the socket resource with syn_flood test
 867		 * as inet_csk_destroy_sock will not be called
 868		 * in tcp_done since SOCK_DEAD flag is not set.
 869		 * Kernel handles this differently where new socket is
 870		 * created only after 3 way handshake is done.
 871		 */
 872		sock_orphan(child);
 873		tcp_orphan_count_inc();
 874		chtls_release_resources(child);
 875		chtls_conn_done(child);
 876	} else {
 877		if (csk_flag(child, CSK_RST_ABORTED)) {
 878			chtls_release_resources(child);
 879			chtls_conn_done(child);
 880		}
 881	}
 882}
 883
 884static void pass_open_abort(struct sock *child, struct sock *parent,
 885			    struct sk_buff *skb)
 886{
 887	do_abort_syn_rcv(child, parent);
 888	kfree_skb(skb);
 889}
 890
 891static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
 892{
 893	pass_open_abort(skb->sk, lsk, skb);
 894}
 895
 896static void chtls_pass_open_arp_failure(struct sock *sk,
 897					struct sk_buff *skb)
 898{
 899	const struct request_sock *oreq;
 900	struct chtls_sock *csk;
 901	struct chtls_dev *cdev;
 902	struct sock *parent;
 903	void *data;
 904
 905	csk = rcu_dereference_sk_user_data(sk);
 906	cdev = csk->cdev;
 907
 908	/*
 909	 * If the connection is being aborted due to the parent listening
 910	 * socket going away there's nothing to do, the ABORT_REQ will close
 911	 * the connection.
 912	 */
 913	if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
 914		kfree_skb(skb);
 915		return;
 916	}
 917
 918	oreq = csk->passive_reap_next;
 919	data = lookup_stid(cdev->tids, oreq->ts_recent);
 920	parent = ((struct listen_ctx *)data)->lsk;
 921
 922	bh_lock_sock(parent);
 923	if (!sock_owned_by_user(parent)) {
 924		pass_open_abort(sk, parent, skb);
 925	} else {
 926		BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
 927		__sk_add_backlog(parent, skb);
 928	}
 929	bh_unlock_sock(parent);
 930}
 931
 932static void chtls_accept_rpl_arp_failure(void *handle,
 933					 struct sk_buff *skb)
 934{
 935	struct sock *sk = (struct sock *)handle;
 936
 937	sock_hold(sk);
 938	process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
 939	sock_put(sk);
 940}
 941
 942static unsigned int chtls_select_mss(const struct chtls_sock *csk,
 943				     unsigned int pmtu,
 944				     struct cpl_pass_accept_req *req)
 945{
 946	struct chtls_dev *cdev;
 947	struct dst_entry *dst;
 948	unsigned int tcpoptsz;
 949	unsigned int iphdrsz;
 950	unsigned int mtu_idx;
 951	struct tcp_sock *tp;
 952	unsigned int mss;
 953	struct sock *sk;
 954	u16 user_mss;
 955
 956	mss = ntohs(req->tcpopt.mss);
 957	sk = csk->sk;
 958	dst = __sk_dst_get(sk);
 959	cdev = csk->cdev;
 960	tp = tcp_sk(sk);
 961	tcpoptsz = 0;
 962
 963#if IS_ENABLED(CONFIG_IPV6)
 964	if (sk->sk_family == AF_INET6)
 965		iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
 966	else
 967#endif
 968		iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
 969	if (req->tcpopt.tstamp)
 970		tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
 971
 972	tp->advmss = dst_metric_advmss(dst);
 973	user_mss = USER_MSS(tp);
 974	if (user_mss && tp->advmss > user_mss)
 975		tp->advmss = user_mss;
 976	if (tp->advmss > pmtu - iphdrsz)
 977		tp->advmss = pmtu - iphdrsz;
 978	if (mss && tp->advmss > mss)
 979		tp->advmss = mss;
 980
 981	tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
 982					    iphdrsz + tcpoptsz,
 983					    tp->advmss - tcpoptsz,
 984					    8, &mtu_idx);
 985	tp->advmss -= iphdrsz;
 986
 987	inet_csk(sk)->icsk_pmtu_cookie = pmtu;
 988	return mtu_idx;
 989}
 990
 991static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
 992{
 993	int wscale = 0;
 994
 995	if (space > MAX_RCV_WND)
 996		space = MAX_RCV_WND;
 997	if (win_clamp && win_clamp < space)
 998		space = win_clamp;
 999
1000	if (wscale_ok) {
1001		while (wscale < 14 && (65535 << wscale) < space)
1002			wscale++;
1003	}
1004	return wscale;
1005}
1006
1007static void chtls_pass_accept_rpl(struct sk_buff *skb,
1008				  struct cpl_pass_accept_req *req,
1009				  unsigned int tid)
1010
1011{
1012	struct cpl_t5_pass_accept_rpl *rpl5;
1013	struct cxgb4_lld_info *lldi;
1014	const struct tcphdr *tcph;
1015	const struct tcp_sock *tp;
1016	struct chtls_sock *csk;
1017	unsigned int len;
1018	struct sock *sk;
1019	u32 opt2, hlen;
1020	u64 opt0;
1021
1022	sk = skb->sk;
1023	tp = tcp_sk(sk);
1024	csk = sk->sk_user_data;
1025	csk->tid = tid;
1026	lldi = csk->cdev->lldi;
1027	len = roundup(sizeof(*rpl5), 16);
1028
1029	rpl5 = __skb_put_zero(skb, len);
1030	INIT_TP_WR(rpl5, tid);
1031
1032	OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1033						     csk->tid));
1034	csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
1035					req);
1036	opt0 = TCAM_BYPASS_F |
1037	       WND_SCALE_V(RCV_WSCALE(tp)) |
1038	       MSS_IDX_V(csk->mtu_idx) |
1039	       L2T_IDX_V(csk->l2t_entry->idx) |
1040	       NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
1041	       TX_CHAN_V(csk->tx_chan) |
1042	       SMAC_SEL_V(csk->smac_idx) |
1043	       DSCP_V(csk->tos >> 2) |
1044	       ULP_MODE_V(ULP_MODE_TLS) |
1045	       RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
1046
1047	opt2 = RX_CHANNEL_V(0) |
1048		RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1049
1050	if (!is_t5(lldi->adapter_type))
1051		opt2 |= RX_FC_DISABLE_F;
1052	if (req->tcpopt.tstamp)
1053		opt2 |= TSTAMPS_EN_F;
1054	if (req->tcpopt.sack)
1055		opt2 |= SACK_EN_F;
1056	hlen = ntohl(req->hdr_len);
1057
1058	tcph = (struct tcphdr *)((u8 *)(req + 1) +
1059			T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1060	if (tcph->ece && tcph->cwr)
1061		opt2 |= CCTRL_ECN_V(1);
1062	opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1063	opt2 |= T5_ISS_F;
1064	opt2 |= T5_OPT_2_VALID_F;
1065	opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
1066	rpl5->opt0 = cpu_to_be64(opt0);
1067	rpl5->opt2 = cpu_to_be32(opt2);
1068	rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
1069	set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
1070	t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
1071	cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
1072}
1073
1074static void inet_inherit_port(struct sock *lsk, struct sock *newsk)
1075{
1076	local_bh_disable();
1077	__inet_inherit_port(lsk, newsk);
1078	local_bh_enable();
1079}
1080
1081static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1082{
1083	if (skb->protocol) {
1084		kfree_skb(skb);
1085		return 0;
1086	}
1087	BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
1088	return 0;
1089}
1090
1091static void chtls_set_tcp_window(struct chtls_sock *csk)
1092{
1093	struct net_device *ndev = csk->egress_dev;
1094	struct port_info *pi = netdev_priv(ndev);
1095	unsigned int linkspeed;
1096	u8 scale;
1097
1098	linkspeed = pi->link_cfg.speed;
1099	scale = linkspeed / SPEED_10000;
1100#define CHTLS_10G_RCVWIN (256 * 1024)
1101	csk->rcv_win = CHTLS_10G_RCVWIN;
1102	if (scale)
1103		csk->rcv_win *= scale;
1104#define CHTLS_10G_SNDWIN (256 * 1024)
1105	csk->snd_win = CHTLS_10G_SNDWIN;
1106	if (scale)
1107		csk->snd_win *= scale;
1108}
1109
1110static struct sock *chtls_recv_sock(struct sock *lsk,
1111				    struct request_sock *oreq,
1112				    void *network_hdr,
1113				    const struct cpl_pass_accept_req *req,
1114				    struct chtls_dev *cdev)
1115{
1116	struct adapter *adap = pci_get_drvdata(cdev->pdev);
1117	struct neighbour *n = NULL;
1118	struct inet_sock *newinet;
1119	const struct iphdr *iph;
1120	struct tls_context *ctx;
1121	struct net_device *ndev;
1122	struct chtls_sock *csk;
1123	struct dst_entry *dst;
1124	struct tcp_sock *tp;
1125	struct sock *newsk;
1126	bool found = false;
1127	u16 port_id;
1128	int rxq_idx;
1129	int step, i;
1130
1131	iph = (const struct iphdr *)network_hdr;
1132	newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1133	if (!newsk)
1134		goto free_oreq;
1135
1136	if (lsk->sk_family == AF_INET) {
1137		dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1138		if (!dst)
1139			goto free_sk;
1140
1141		n = dst_neigh_lookup(dst, &iph->saddr);
1142#if IS_ENABLED(CONFIG_IPV6)
1143	} else {
1144		const struct ipv6hdr *ip6h;
1145		struct flowi6 fl6;
1146
1147		ip6h = (const struct ipv6hdr *)network_hdr;
1148		memset(&fl6, 0, sizeof(fl6));
1149		fl6.flowi6_proto = IPPROTO_TCP;
1150		fl6.saddr = ip6h->daddr;
1151		fl6.daddr = ip6h->saddr;
1152		fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
1153		fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
1154		security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6));
1155		dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
1156		if (IS_ERR(dst))
1157			goto free_sk;
1158		n = dst_neigh_lookup(dst, &ip6h->saddr);
1159#endif
1160	}
1161	if (!n || !n->dev)
1162		goto free_dst;
1163
1164	ndev = n->dev;
1165	if (is_vlan_dev(ndev))
1166		ndev = vlan_dev_real_dev(ndev);
1167
1168	for_each_port(adap, i)
1169		if (cdev->ports[i] == ndev)
1170			found = true;
1171
1172	if (!found)
1173		goto free_dst;
1174
1175	port_id = cxgb4_port_idx(ndev);
1176
1177	csk = chtls_sock_create(cdev);
1178	if (!csk)
1179		goto free_dst;
1180
1181	csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1182	if (!csk->l2t_entry)
1183		goto free_csk;
1184
1185	newsk->sk_user_data = csk;
1186	newsk->sk_backlog_rcv = chtls_backlog_rcv;
1187
1188	tp = tcp_sk(newsk);
1189	newinet = inet_sk(newsk);
1190
1191	if (iph->version == 0x4) {
1192		newinet->inet_daddr = iph->saddr;
1193		newinet->inet_rcv_saddr = iph->daddr;
1194		newinet->inet_saddr = iph->daddr;
1195#if IS_ENABLED(CONFIG_IPV6)
1196	} else {
1197		struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
1198		struct inet_request_sock *treq = inet_rsk(oreq);
1199		struct ipv6_pinfo *newnp = inet6_sk(newsk);
1200		struct ipv6_pinfo *np = inet6_sk(lsk);
1201
1202		newinet->pinet6 = &newtcp6sk->inet6;
1203		newinet->ipv6_fl_list = NULL;
1204		memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1205		newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
1206		newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
1207		inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
1208		newnp->pktoptions = NULL;
1209		newsk->sk_bound_dev_if = treq->ir_iif;
1210		newinet->inet_opt = NULL;
1211		newinet->inet_daddr = LOOPBACK4_IPV6;
1212		newinet->inet_saddr = LOOPBACK4_IPV6;
1213#endif
1214	}
1215
1216	oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1217	sk_setup_caps(newsk, dst);
1218	ctx = tls_get_ctx(lsk);
1219	newsk->sk_destruct = ctx->sk_destruct;
1220	newsk->sk_prot_creator = lsk->sk_prot_creator;
1221	csk->sk = newsk;
1222	csk->passive_reap_next = oreq;
1223	csk->tx_chan = cxgb4_port_chan(ndev);
1224	csk->port_id = port_id;
1225	csk->egress_dev = ndev;
1226	csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1227	chtls_set_tcp_window(csk);
1228	tp->rcv_wnd = csk->rcv_win;
1229	csk->sndbuf = csk->snd_win;
1230	csk->ulp_mode = ULP_MODE_TLS;
1231	step = cdev->lldi->nrxq / cdev->lldi->nchan;
1232	rxq_idx = port_id * step;
1233	rxq_idx += cdev->round_robin_cnt++ % step;
1234	csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx];
1235	csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1236			port_id * step;
1237	csk->sndbuf = newsk->sk_sndbuf;
1238	csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
1239	RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
1240					   READ_ONCE(sock_net(newsk)->
1241						     ipv4.sysctl_tcp_window_scaling),
1242					   tp->window_clamp);
1243	neigh_release(n);
1244	inet_inherit_port(lsk, newsk);
1245	csk_set_flag(csk, CSK_CONN_INLINE);
1246	bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1247
1248	return newsk;
1249free_csk:
1250	chtls_sock_release(&csk->kref);
1251free_dst:
1252	if (n)
1253		neigh_release(n);
1254	dst_release(dst);
1255free_sk:
1256	inet_csk_prepare_forced_close(newsk);
1257	tcp_done(newsk);
1258free_oreq:
1259	chtls_reqsk_free(oreq);
1260	return NULL;
1261}
1262
1263/*
1264 * Populate a TID_RELEASE WR.  The skb must be already propely sized.
1265 */
1266static  void mk_tid_release(struct sk_buff *skb,
1267			    unsigned int chan, unsigned int tid)
1268{
1269	struct cpl_tid_release *req;
1270	unsigned int len;
1271
1272	len = roundup(sizeof(struct cpl_tid_release), 16);
1273	req = (struct cpl_tid_release *)__skb_put(skb, len);
1274	memset(req, 0, len);
1275	set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1276	INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1277}
1278
1279static int chtls_get_module(struct sock *sk)
1280{
1281	struct inet_connection_sock *icsk = inet_csk(sk);
1282
1283	if (!try_module_get(icsk->icsk_ulp_ops->owner))
1284		return -1;
1285
1286	return 0;
1287}
1288
1289static void chtls_pass_accept_request(struct sock *sk,
1290				      struct sk_buff *skb)
1291{
1292	struct cpl_t5_pass_accept_rpl *rpl;
1293	struct cpl_pass_accept_req *req;
1294	struct listen_ctx *listen_ctx;
1295	struct vlan_ethhdr *vlan_eh;
1296	struct request_sock *oreq;
1297	struct sk_buff *reply_skb;
1298	struct chtls_sock *csk;
1299	struct chtls_dev *cdev;
1300	struct ipv6hdr *ip6h;
1301	struct tcphdr *tcph;
1302	struct sock *newsk;
1303	struct ethhdr *eh;
1304	struct iphdr *iph;
1305	void *network_hdr;
1306	unsigned int stid;
1307	unsigned int len;
1308	unsigned int tid;
1309	bool th_ecn, ect;
1310	__u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1311	u16 eth_hdr_len;
1312	bool ecn_ok;
1313
1314	req = cplhdr(skb) + RSS_HDR;
1315	tid = GET_TID(req);
1316	cdev = BLOG_SKB_CB(skb)->cdev;
1317	newsk = lookup_tid(cdev->tids, tid);
1318	stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1319	if (newsk) {
1320		pr_info("tid (%d) already in use\n", tid);
1321		return;
1322	}
1323
1324	len = roundup(sizeof(*rpl), 16);
1325	reply_skb = alloc_skb(len, GFP_ATOMIC);
1326	if (!reply_skb) {
1327		cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1328		kfree_skb(skb);
1329		return;
1330	}
1331
1332	if (sk->sk_state != TCP_LISTEN)
1333		goto reject;
1334
1335	if (inet_csk_reqsk_queue_is_full(sk))
1336		goto reject;
1337
1338	if (sk_acceptq_is_full(sk))
1339		goto reject;
1340
1341
1342	eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1343	if (eth_hdr_len == ETH_HLEN) {
1344		eh = (struct ethhdr *)(req + 1);
1345		iph = (struct iphdr *)(eh + 1);
1346		ip6h = (struct ipv6hdr *)(eh + 1);
1347		network_hdr = (void *)(eh + 1);
1348	} else {
1349		vlan_eh = (struct vlan_ethhdr *)(req + 1);
1350		iph = (struct iphdr *)(vlan_eh + 1);
1351		ip6h = (struct ipv6hdr *)(vlan_eh + 1);
1352		network_hdr = (void *)(vlan_eh + 1);
1353	}
1354
1355	if (iph->version == 0x4) {
1356		tcph = (struct tcphdr *)(iph + 1);
1357		skb_set_network_header(skb, (void *)iph - (void *)req);
1358		oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1359	} else {
1360		tcph = (struct tcphdr *)(ip6h + 1);
1361		skb_set_network_header(skb, (void *)ip6h - (void *)req);
1362		oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
1363	}
1364
1365	if (!oreq)
1366		goto reject;
1367
1368	oreq->rsk_rcv_wnd = 0;
1369	oreq->rsk_window_clamp = 0;
1370	oreq->syncookie = 0;
1371	oreq->mss = 0;
1372	oreq->ts_recent = 0;
1373
1374	tcp_rsk(oreq)->tfo_listener = false;
1375	tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1376	chtls_set_req_port(oreq, tcph->source, tcph->dest);
1377	if (iph->version == 0x4) {
1378		chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1379		ip_dsfield = ipv4_get_dsfield(iph);
1380#if IS_ENABLED(CONFIG_IPV6)
1381	} else {
1382		inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
1383		inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
1384		ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
1385#endif
1386	}
1387	if (req->tcpopt.wsf <= 14 &&
1388	    READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
1389		inet_rsk(oreq)->wscale_ok = 1;
1390		inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1391	}
1392	inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
1393	th_ecn = tcph->ece && tcph->cwr;
1394	if (th_ecn) {
1395		ect = !INET_ECN_is_not_ect(ip_dsfield);
1396		ecn_ok = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn);
1397		if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1398			inet_rsk(oreq)->ecn_ok = 1;
1399	}
1400
1401	newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1402	if (!newsk)
1403		goto reject;
1404
1405	if (chtls_get_module(newsk))
1406		goto reject;
1407	inet_csk_reqsk_queue_added(sk);
1408	reply_skb->sk = newsk;
1409	chtls_install_cpl_ops(newsk);
1410	cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1411	csk = rcu_dereference_sk_user_data(newsk);
1412	listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1413	csk->listen_ctx = listen_ctx;
1414	__skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1415	chtls_pass_accept_rpl(reply_skb, req, tid);
1416	kfree_skb(skb);
1417	return;
1418
1419reject:
1420	mk_tid_release(reply_skb, 0, tid);
1421	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1422	kfree_skb(skb);
1423}
1424
1425/*
1426 * Handle a CPL_PASS_ACCEPT_REQ message.
1427 */
1428static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1429{
1430	struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1431	struct listen_ctx *ctx;
1432	unsigned int stid;
1433	unsigned int tid;
1434	struct sock *lsk;
1435	void *data;
1436
1437	stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1438	tid = GET_TID(req);
1439
1440	data = lookup_stid(cdev->tids, stid);
1441	if (!data)
1442		return 1;
1443
1444	ctx = (struct listen_ctx *)data;
1445	lsk = ctx->lsk;
1446
1447	if (unlikely(tid_out_of_range(cdev->tids, tid))) {
1448		pr_info("passive open TID %u too large\n", tid);
1449		return 1;
1450	}
1451
1452	BLOG_SKB_CB(skb)->cdev = cdev;
1453	process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1454	return 0;
1455}
1456
1457/*
1458 * Completes some final bits of initialization for just established connections
1459 * and changes their state to TCP_ESTABLISHED.
1460 *
1461 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1462 */
1463static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1464{
1465	struct tcp_sock *tp = tcp_sk(sk);
1466
1467	tp->pushed_seq = snd_isn;
1468	tp->write_seq = snd_isn;
1469	tp->snd_nxt = snd_isn;
1470	tp->snd_una = snd_isn;
1471	atomic_set(&inet_sk(sk)->inet_id, get_random_u16());
1472	assign_rxopt(sk, opt);
1473
1474	if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1475		tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1476
1477	smp_mb();
1478	tcp_set_state(sk, TCP_ESTABLISHED);
1479}
1480
1481static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1482{
1483	struct sk_buff *abort_skb;
1484
1485	abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1486	if (abort_skb)
1487		chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1488}
1489
1490static struct sock *reap_list;
1491static DEFINE_SPINLOCK(reap_list_lock);
1492
1493/*
1494 * Process the reap list.
1495 */
1496DECLARE_TASK_FUNC(process_reap_list, task_param)
1497{
1498	spin_lock_bh(&reap_list_lock);
1499	while (reap_list) {
1500		struct sock *sk = reap_list;
1501		struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1502
1503		reap_list = csk->passive_reap_next;
1504		csk->passive_reap_next = NULL;
1505		spin_unlock(&reap_list_lock);
1506		sock_hold(sk);
1507
1508		bh_lock_sock(sk);
1509		chtls_abort_conn(sk, NULL);
1510		sock_orphan(sk);
1511		if (sk->sk_state == TCP_CLOSE)
1512			inet_csk_destroy_sock(sk);
1513		bh_unlock_sock(sk);
1514		sock_put(sk);
1515		spin_lock(&reap_list_lock);
1516	}
1517	spin_unlock_bh(&reap_list_lock);
1518}
1519
1520static DECLARE_WORK(reap_task, process_reap_list);
1521
1522static void add_to_reap_list(struct sock *sk)
1523{
1524	struct chtls_sock *csk = sk->sk_user_data;
1525
1526	local_bh_disable();
1527	release_tcp_port(sk); /* release the port immediately */
1528
1529	spin_lock(&reap_list_lock);
1530	csk->passive_reap_next = reap_list;
1531	reap_list = sk;
1532	if (!csk->passive_reap_next)
1533		schedule_work(&reap_task);
1534	spin_unlock(&reap_list_lock);
1535	local_bh_enable();
1536}
1537
1538static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1539				    struct chtls_dev *cdev)
1540{
1541	struct request_sock *oreq;
1542	struct chtls_sock *csk;
1543
1544	if (lsk->sk_state != TCP_LISTEN)
1545		return;
1546
1547	csk = child->sk_user_data;
1548	oreq = csk->passive_reap_next;
1549	csk->passive_reap_next = NULL;
1550
1551	reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1552	__skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1553
1554	if (sk_acceptq_is_full(lsk)) {
1555		chtls_reqsk_free(oreq);
1556		add_to_reap_list(child);
1557	} else {
1558		refcount_set(&oreq->rsk_refcnt, 1);
1559		inet_csk_reqsk_queue_add(lsk, oreq, child);
1560		lsk->sk_data_ready(lsk);
1561	}
1562}
1563
1564static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1565{
1566	struct sock *child = skb->sk;
1567
1568	skb->sk = NULL;
1569	add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1570	kfree_skb(skb);
1571}
1572
1573static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1574{
1575	struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1576	struct chtls_sock *csk;
1577	struct sock *lsk, *sk;
1578	unsigned int hwtid;
1579
1580	hwtid = GET_TID(req);
1581	sk = lookup_tid(cdev->tids, hwtid);
1582	if (!sk)
1583		return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1584
1585	bh_lock_sock(sk);
1586	if (unlikely(sock_owned_by_user(sk))) {
1587		kfree_skb(skb);
1588	} else {
1589		unsigned int stid;
1590		void *data;
1591
1592		csk = sk->sk_user_data;
1593		csk->wr_max_credits = 64;
1594		csk->wr_credits = 64;
1595		csk->wr_unacked = 0;
1596		make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1597		stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1598		sk->sk_state_change(sk);
1599		if (unlikely(sk->sk_socket))
1600			sk_wake_async(sk, 0, POLL_OUT);
1601
1602		data = lookup_stid(cdev->tids, stid);
1603		if (!data) {
1604			/* listening server close */
1605			kfree_skb(skb);
1606			goto unlock;
1607		}
1608		lsk = ((struct listen_ctx *)data)->lsk;
1609
1610		bh_lock_sock(lsk);
1611		if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1612			/* removed from synq */
1613			bh_unlock_sock(lsk);
1614			kfree_skb(skb);
1615			goto unlock;
1616		}
1617
1618		if (likely(!sock_owned_by_user(lsk))) {
1619			kfree_skb(skb);
1620			add_pass_open_to_parent(sk, lsk, cdev);
1621		} else {
1622			skb->sk = sk;
1623			BLOG_SKB_CB(skb)->cdev = cdev;
1624			BLOG_SKB_CB(skb)->backlog_rcv =
1625				bl_add_pass_open_to_parent;
1626			__sk_add_backlog(lsk, skb);
1627		}
1628		bh_unlock_sock(lsk);
1629	}
1630unlock:
1631	bh_unlock_sock(sk);
1632	return 0;
1633}
1634
1635/*
1636 * Handle receipt of an urgent pointer.
1637 */
1638static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1639{
1640	struct tcp_sock *tp = tcp_sk(sk);
1641
1642	urg_seq--;
1643	if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1644		return;	/* duplicate pointer */
1645
1646	sk_send_sigurg(sk);
1647	if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1648	    !sock_flag(sk, SOCK_URGINLINE) &&
1649	    tp->copied_seq != tp->rcv_nxt) {
1650		struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1651
1652		tp->copied_seq++;
1653		if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1654			chtls_free_skb(sk, skb);
1655	}
1656
1657	tp->urg_data = TCP_URG_NOTYET;
1658	tp->urg_seq = urg_seq;
1659}
1660
1661static void check_sk_callbacks(struct chtls_sock *csk)
1662{
1663	struct sock *sk = csk->sk;
1664
1665	if (unlikely(sk->sk_user_data &&
1666		     !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1667		csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1668}
1669
1670/*
1671 * Handles Rx data that arrives in a state where the socket isn't accepting
1672 * new data.
1673 */
1674static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1675{
1676	if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1677		chtls_abort_conn(sk, skb);
1678
1679	kfree_skb(skb);
1680}
1681
1682static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1683{
1684	struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1685	struct chtls_sock *csk;
1686	struct tcp_sock *tp;
1687
1688	csk = rcu_dereference_sk_user_data(sk);
1689	tp = tcp_sk(sk);
1690
1691	if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1692		handle_excess_rx(sk, skb);
1693		return;
1694	}
1695
1696	ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1697	ULP_SKB_CB(skb)->psh = hdr->psh;
1698	skb_ulp_mode(skb) = ULP_MODE_NONE;
1699
1700	skb_reset_transport_header(skb);
1701	__skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1702	if (!skb->data_len)
1703		__skb_trim(skb, ntohs(hdr->len));
1704
1705	if (unlikely(hdr->urg))
1706		handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1707	if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1708		     tp->urg_seq - tp->rcv_nxt < skb->len))
1709		tp->urg_data = TCP_URG_VALID |
1710			       skb->data[tp->urg_seq - tp->rcv_nxt];
1711
1712	if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1713		csk->delack_mode = hdr->dack_mode;
1714		csk->delack_seq = tp->rcv_nxt;
1715	}
1716
1717	tcp_hdr(skb)->fin = 0;
1718	tp->rcv_nxt += skb->len;
1719
1720	__skb_queue_tail(&sk->sk_receive_queue, skb);
1721
1722	if (!sock_flag(sk, SOCK_DEAD)) {
1723		check_sk_callbacks(csk);
1724		sk->sk_data_ready(sk);
1725	}
1726}
1727
1728static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1729{
1730	struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1731	unsigned int hwtid = GET_TID(req);
1732	struct sock *sk;
1733
1734	sk = lookup_tid(cdev->tids, hwtid);
1735	if (unlikely(!sk)) {
1736		pr_err("can't find conn. for hwtid %u.\n", hwtid);
1737		return -EINVAL;
1738	}
1739	skb_dstref_steal(skb);
1740	process_cpl_msg(chtls_recv_data, sk, skb);
1741	return 0;
1742}
1743
1744static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1745{
1746	struct cpl_tls_data *hdr = cplhdr(skb);
1747	struct chtls_sock *csk;
1748	struct chtls_hws *tlsk;
1749	struct tcp_sock *tp;
1750
1751	csk = rcu_dereference_sk_user_data(sk);
1752	tlsk = &csk->tlshws;
1753	tp = tcp_sk(sk);
1754
1755	if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1756		handle_excess_rx(sk, skb);
1757		return;
1758	}
1759
1760	ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1761	ULP_SKB_CB(skb)->flags = 0;
1762	skb_ulp_mode(skb) = ULP_MODE_TLS;
1763
1764	skb_reset_transport_header(skb);
1765	__skb_pull(skb, sizeof(*hdr));
1766	if (!skb->data_len)
1767		__skb_trim(skb,
1768			   CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1769
1770	if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1771		     tp->rcv_nxt < skb->len))
1772		tp->urg_data = TCP_URG_VALID |
1773			       skb->data[tp->urg_seq - tp->rcv_nxt];
1774
1775	tcp_hdr(skb)->fin = 0;
1776	tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1777	__skb_queue_tail(&tlsk->sk_recv_queue, skb);
1778}
1779
1780static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1781{
1782	struct cpl_tls_data *req = cplhdr(skb);
1783	unsigned int hwtid = GET_TID(req);
1784	struct sock *sk;
1785
1786	sk = lookup_tid(cdev->tids, hwtid);
1787	if (unlikely(!sk)) {
1788		pr_err("can't find conn. for hwtid %u.\n", hwtid);
1789		return -EINVAL;
1790	}
1791	skb_dstref_steal(skb);
1792	process_cpl_msg(chtls_recv_pdu, sk, skb);
1793	return 0;
1794}
1795
1796static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1797{
1798	struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1799
1800	skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1801	tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1802}
1803
1804static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1805{
1806	struct tlsrx_cmp_hdr *tls_hdr_pkt;
1807	struct cpl_rx_tls_cmp *cmp_cpl;
1808	struct sk_buff *skb_rec;
1809	struct chtls_sock *csk;
1810	struct chtls_hws *tlsk;
1811	struct tcp_sock *tp;
1812
1813	cmp_cpl = cplhdr(skb);
1814	csk = rcu_dereference_sk_user_data(sk);
1815	tlsk = &csk->tlshws;
1816	tp = tcp_sk(sk);
1817
1818	ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1819	ULP_SKB_CB(skb)->flags = 0;
1820
1821	skb_reset_transport_header(skb);
1822	__skb_pull(skb, sizeof(*cmp_cpl));
1823	tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1824	if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1825		tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1826	if (!skb->data_len)
1827		__skb_trim(skb, TLS_HEADER_LENGTH);
1828
1829	tp->rcv_nxt +=
1830		CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1831
1832	ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
1833	skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1834	if (!skb_rec) {
1835		__skb_queue_tail(&sk->sk_receive_queue, skb);
1836	} else {
1837		chtls_set_hdrlen(skb, tlsk->pldlen);
1838		tlsk->pldlen = 0;
1839		__skb_queue_tail(&sk->sk_receive_queue, skb);
1840		__skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1841	}
1842
1843	if (!sock_flag(sk, SOCK_DEAD)) {
1844		check_sk_callbacks(csk);
1845		sk->sk_data_ready(sk);
1846	}
1847}
1848
1849static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1850{
1851	struct cpl_rx_tls_cmp *req = cplhdr(skb);
1852	unsigned int hwtid = GET_TID(req);
1853	struct sock *sk;
1854
1855	sk = lookup_tid(cdev->tids, hwtid);
1856	if (unlikely(!sk)) {
1857		pr_err("can't find conn. for hwtid %u.\n", hwtid);
1858		return -EINVAL;
1859	}
1860	skb_dstref_steal(skb);
1861	process_cpl_msg(chtls_rx_hdr, sk, skb);
1862
1863	return 0;
1864}
1865
1866static void chtls_timewait(struct sock *sk)
1867{
1868	struct tcp_sock *tp = tcp_sk(sk);
1869
1870	tp->rcv_nxt++;
1871	tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
1872	tp->srtt_us = 0;
1873	tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1874}
1875
1876static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1877{
1878	struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1879
1880	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1881		goto out;
1882
1883	sk->sk_shutdown |= RCV_SHUTDOWN;
1884	sock_set_flag(sk, SOCK_DONE);
1885
1886	switch (sk->sk_state) {
1887	case TCP_SYN_RECV:
1888	case TCP_ESTABLISHED:
1889		tcp_set_state(sk, TCP_CLOSE_WAIT);
1890		break;
1891	case TCP_FIN_WAIT1:
1892		tcp_set_state(sk, TCP_CLOSING);
1893		break;
1894	case TCP_FIN_WAIT2:
1895		chtls_release_resources(sk);
1896		if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1897			chtls_conn_done(sk);
1898		else
1899			chtls_timewait(sk);
1900		break;
1901	default:
1902		pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1903	}
1904
1905	if (!sock_flag(sk, SOCK_DEAD)) {
1906		sk->sk_state_change(sk);
1907		/* Do not send POLL_HUP for half duplex close. */
1908
1909		if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1910		    sk->sk_state == TCP_CLOSE)
1911			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1912		else
1913			sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1914	}
1915out:
1916	kfree_skb(skb);
1917}
1918
1919static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1920{
1921	struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1922	struct chtls_sock *csk;
1923	struct tcp_sock *tp;
1924
1925	csk = rcu_dereference_sk_user_data(sk);
1926
1927	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1928		goto out;
1929
1930	tp = tcp_sk(sk);
1931
1932	tp->snd_una = ntohl(rpl->snd_nxt) - 1;  /* exclude FIN */
1933
1934	switch (sk->sk_state) {
1935	case TCP_CLOSING:
1936		chtls_release_resources(sk);
1937		if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1938			chtls_conn_done(sk);
1939		else
1940			chtls_timewait(sk);
1941		break;
1942	case TCP_LAST_ACK:
1943		chtls_release_resources(sk);
1944		chtls_conn_done(sk);
1945		break;
1946	case TCP_FIN_WAIT1:
1947		tcp_set_state(sk, TCP_FIN_WAIT2);
1948		sk->sk_shutdown |= SEND_SHUTDOWN;
1949
1950		if (!sock_flag(sk, SOCK_DEAD))
1951			sk->sk_state_change(sk);
1952		else if (tcp_sk(sk)->linger2 < 0 &&
1953			 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1954			chtls_abort_conn(sk, skb);
1955		else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT))
1956			chtls_set_quiesce_ctrl(sk, 0);
1957		break;
1958	default:
1959		pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1960	}
1961out:
1962	kfree_skb(skb);
1963}
1964
1965static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1966				   size_t len, gfp_t gfp)
1967{
1968	if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1969		WARN_ONCE(skb->len < len, "skb alloc error");
1970		__skb_trim(skb, len);
1971		skb_get(skb);
1972	} else {
1973		skb = alloc_skb(len, gfp);
1974		if (skb)
1975			__skb_put(skb, len);
1976	}
1977	return skb;
1978}
1979
1980static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1981			     int cmd)
1982{
1983	struct cpl_abort_rpl *rpl = cplhdr(skb);
1984
1985	INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1986	rpl->cmd = cmd;
1987}
1988
1989static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1990{
1991	struct cpl_abort_req_rss *req = cplhdr(skb);
1992	struct sk_buff *reply_skb;
1993
1994	reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1995			      GFP_KERNEL | __GFP_NOFAIL);
1996	__skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1997	set_abort_rpl_wr(reply_skb, GET_TID(req),
1998			 (req->status & CPL_ABORT_NO_RST));
1999	set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
2000	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2001	kfree_skb(skb);
2002}
2003
2004/*
2005 * Add an skb to the deferred skb queue for processing from process context.
2006 */
2007static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
2008			   defer_handler_t handler)
2009{
2010	DEFERRED_SKB_CB(skb)->handler = handler;
2011	spin_lock_bh(&cdev->deferq.lock);
2012	__skb_queue_tail(&cdev->deferq, skb);
2013	if (skb_queue_len(&cdev->deferq) == 1)
2014		schedule_work(&cdev->deferq_task);
2015	spin_unlock_bh(&cdev->deferq.lock);
2016}
2017
2018static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
2019				 struct chtls_dev *cdev,
2020				 int status, int queue)
2021{
2022	struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2023	struct sk_buff *reply_skb;
2024	struct chtls_sock *csk;
2025	unsigned int tid;
2026
2027	csk = rcu_dereference_sk_user_data(sk);
2028	tid = GET_TID(req);
2029
2030	reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
2031	if (!reply_skb) {
2032		req->status = (queue << 1) | status;
2033		t4_defer_reply(skb, cdev, send_defer_abort_rpl);
2034		return;
2035	}
2036
2037	set_abort_rpl_wr(reply_skb, tid, status);
2038	kfree_skb(skb);
2039	set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
2040	if (csk_conn_inline(csk)) {
2041		struct l2t_entry *e = csk->l2t_entry;
2042
2043		if (e && sk->sk_state != TCP_SYN_RECV) {
2044			cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2045			return;
2046		}
2047	}
2048	cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2049}
2050
2051/*
2052 * This is run from a listener's backlog to abort a child connection in
2053 * SYN_RCV state (i.e., one on the listener's SYN queue).
2054 */
2055static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
2056{
2057	struct chtls_sock *csk;
2058	struct sock *child;
2059	int queue;
2060
2061	child = skb->sk;
2062	csk = rcu_dereference_sk_user_data(child);
2063	queue = csk->txq_idx;
2064
2065	skb->sk	= NULL;
2066	chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
2067			     CPL_ABORT_NO_RST, queue);
2068	do_abort_syn_rcv(child, lsk);
2069}
2070
2071static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
2072{
2073	const struct request_sock *oreq;
2074	struct listen_ctx *listen_ctx;
2075	struct chtls_sock *csk;
2076	struct chtls_dev *cdev;
2077	struct sock *psk;
2078	void *ctx;
2079
2080	csk = sk->sk_user_data;
2081	oreq = csk->passive_reap_next;
2082	cdev = csk->cdev;
2083
2084	if (!oreq)
2085		return -1;
2086
2087	ctx = lookup_stid(cdev->tids, oreq->ts_recent);
2088	if (!ctx)
2089		return -1;
2090
2091	listen_ctx = (struct listen_ctx *)ctx;
2092	psk = listen_ctx->lsk;
2093
2094	bh_lock_sock(psk);
2095	if (!sock_owned_by_user(psk)) {
2096		int queue = csk->txq_idx;
2097
2098		chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
2099		do_abort_syn_rcv(sk, psk);
2100	} else {
2101		skb->sk = sk;
2102		BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
2103		__sk_add_backlog(psk, skb);
2104	}
2105	bh_unlock_sock(psk);
2106	return 0;
2107}
2108
2109static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
2110{
2111	const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2112	struct chtls_sock *csk = sk->sk_user_data;
2113	int rst_status = CPL_ABORT_NO_RST;
2114	int queue = csk->txq_idx;
2115
2116	if (is_neg_adv(req->status)) {
2117		kfree_skb(skb);
2118		return;
2119	}
2120
2121	csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
2122
2123	if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
2124	    !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
2125		struct tcp_sock *tp = tcp_sk(sk);
2126
2127		if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
2128			WARN_ONCE(1, "send_tx_flowc error");
2129		csk_set_flag(csk, CSK_TX_DATA_SENT);
2130	}
2131
2132	csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
2133
2134	if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2135		sk->sk_err = ETIMEDOUT;
2136
2137		if (!sock_flag(sk, SOCK_DEAD))
2138			sk_error_report(sk);
2139
2140		if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
2141			return;
2142
2143	}
2144
2145	chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
2146			     rst_status, queue);
2147	chtls_release_resources(sk);
2148	chtls_conn_done(sk);
2149}
2150
2151static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2152{
2153	struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2154	struct chtls_sock *csk;
2155	struct chtls_dev *cdev;
2156
2157	csk = rcu_dereference_sk_user_data(sk);
2158	cdev = csk->cdev;
2159
2160	if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2161		csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2162		if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2163			if (sk->sk_state == TCP_SYN_SENT) {
2164				cxgb4_remove_tid(cdev->tids,
2165						 csk->port_id,
2166						 GET_TID(rpl),
2167						 sk->sk_family);
2168				sock_put(sk);
2169			}
2170			chtls_release_resources(sk);
2171			chtls_conn_done(sk);
2172		}
2173	}
2174	kfree_skb(skb);
2175}
2176
2177static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2178{
2179	struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2180	void (*fn)(struct sock *sk, struct sk_buff *skb);
2181	unsigned int hwtid = GET_TID(req);
2182	struct chtls_sock *csk;
2183	struct sock *sk;
2184	u8 opcode;
2185
2186	opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2187
2188	sk = lookup_tid(cdev->tids, hwtid);
2189	if (!sk)
2190		goto rel_skb;
2191
2192	csk = sk->sk_user_data;
2193
2194	switch (opcode) {
2195	case CPL_PEER_CLOSE:
2196		fn = chtls_peer_close;
2197		break;
2198	case CPL_CLOSE_CON_RPL:
2199		fn = chtls_close_con_rpl;
2200		break;
2201	case CPL_ABORT_REQ_RSS:
2202		/*
2203		 * Save the offload device in the skb, we may process this
2204		 * message after the socket has closed.
2205		 */
2206		BLOG_SKB_CB(skb)->cdev = csk->cdev;
2207		fn = chtls_abort_req_rss;
2208		break;
2209	case CPL_ABORT_RPL_RSS:
2210		fn = chtls_abort_rpl_rss;
2211		break;
2212	default:
2213		goto rel_skb;
2214	}
2215
2216	process_cpl_msg(fn, sk, skb);
2217	return 0;
2218
2219rel_skb:
2220	kfree_skb(skb);
2221	return 0;
2222}
2223
2224static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2225{
2226	struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2227	struct chtls_sock *csk = sk->sk_user_data;
2228	struct tcp_sock *tp = tcp_sk(sk);
2229	u32 credits = hdr->credits;
2230	u32 snd_una;
2231
2232	snd_una = ntohl(hdr->snd_una);
2233	csk->wr_credits += credits;
2234
2235	if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2236		csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2237
2238	while (credits) {
2239		struct sk_buff *pskb = csk->wr_skb_head;
2240		u32 csum;
2241
2242		if (unlikely(!pskb)) {
2243			if (csk->wr_nondata)
2244				csk->wr_nondata -= credits;
2245			break;
2246		}
2247		csum = (__force u32)pskb->csum;
2248		if (unlikely(credits < csum)) {
2249			pskb->csum = (__force __wsum)(csum - credits);
2250			break;
2251		}
2252		dequeue_wr(sk);
2253		credits -= csum;
2254		kfree_skb(pskb);
2255	}
2256	if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2257		if (unlikely(before(snd_una, tp->snd_una))) {
2258			kfree_skb(skb);
2259			return;
2260		}
2261
2262		if (tp->snd_una != snd_una) {
2263			tp->snd_una = snd_una;
2264			tp->rcv_tstamp = tcp_jiffies32;
2265			if (tp->snd_una == tp->snd_nxt &&
2266			    !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2267				csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2268		}
2269	}
2270
2271	if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2272		unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2273
2274		csk->wr_credits -= fclen16;
2275		csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2276		csk_reset_flag(csk, CSK_TX_FAILOVER);
2277	}
2278	if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2279		sk->sk_write_space(sk);
2280
2281	kfree_skb(skb);
2282}
2283
2284static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2285{
2286	struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2287	unsigned int hwtid = GET_TID(rpl);
2288	struct sock *sk;
2289
2290	sk = lookup_tid(cdev->tids, hwtid);
2291	if (unlikely(!sk)) {
2292		pr_err("can't find conn. for hwtid %u.\n", hwtid);
2293		return -EINVAL;
2294	}
2295	process_cpl_msg(chtls_rx_ack, sk, skb);
2296
2297	return 0;
2298}
2299
2300static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
2301{
2302	struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR;
2303	unsigned int hwtid = GET_TID(rpl);
2304	struct sock *sk;
2305
2306	sk = lookup_tid(cdev->tids, hwtid);
2307
2308	/* return EINVAL if socket doesn't exist */
2309	if (!sk)
2310		return -EINVAL;
2311
2312	/* Reusing the skb as size of cpl_set_tcb_field structure
2313	 * is greater than cpl_abort_req
2314	 */
2315	if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG)
2316		chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL);
2317
2318	kfree_skb(skb);
2319	return 0;
2320}
2321
2322chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2323	[CPL_PASS_OPEN_RPL]     = chtls_pass_open_rpl,
2324	[CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2325	[CPL_PASS_ACCEPT_REQ]   = chtls_pass_accept_req,
2326	[CPL_PASS_ESTABLISH]    = chtls_pass_establish,
2327	[CPL_RX_DATA]           = chtls_rx_data,
2328	[CPL_TLS_DATA]          = chtls_rx_pdu,
2329	[CPL_RX_TLS_CMP]        = chtls_rx_cmp,
2330	[CPL_PEER_CLOSE]        = chtls_conn_cpl,
2331	[CPL_CLOSE_CON_RPL]     = chtls_conn_cpl,
2332	[CPL_ABORT_REQ_RSS]     = chtls_conn_cpl,
2333	[CPL_ABORT_RPL_RSS]     = chtls_conn_cpl,
2334	[CPL_FW4_ACK]		= chtls_wr_ack,
2335	[CPL_SET_TCB_RPL]	= chtls_set_tcb_rpl,
2336};