chcr_ktls.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
   3
   4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6#include <linux/skbuff.h>
   7#include <linux/module.h>
   8#include <linux/highmem.h>
   9#include <linux/ip.h>
  10#include <net/ipv6.h>
  11#include <linux/netdevice.h>
  12#include <crypto/aes.h>
  13#include <linux/skbuff_ref.h>
  14#include "chcr_ktls.h"
  15
  16static LIST_HEAD(uld_ctx_list);
  17static DEFINE_MUTEX(dev_mutex);
  18
  19/* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
  20 * @skb: skb
  21 * @start: start offset.
  22 * @len: how much data to send after @start
  23 */
  24static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
  25{
  26	struct skb_shared_info *si = skb_shinfo(skb);
  27	u32 frag_size, skb_linear_data_len = skb_headlen(skb);
  28	u8 nfrags = 0, frag_idx = 0;
  29	skb_frag_t *frag;
  30
  31	/* if its a linear skb then return 1 */
  32	if (!skb_is_nonlinear(skb))
  33		return 1;
  34
  35	if (unlikely(start < skb_linear_data_len)) {
  36		frag_size = min(len, skb_linear_data_len - start);
  37	} else {
  38		start -= skb_linear_data_len;
  39
  40		frag = &si->frags[frag_idx];
  41		frag_size = skb_frag_size(frag);
  42		while (start >= frag_size) {
  43			start -= frag_size;
  44			frag_idx++;
  45			frag = &si->frags[frag_idx];
  46			frag_size = skb_frag_size(frag);
  47		}
  48		frag_size = min(len, skb_frag_size(frag) - start);
  49	}
  50	len -= frag_size;
  51	nfrags++;
  52
  53	while (len) {
  54		frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
  55		len -= frag_size;
  56		nfrags++;
  57		frag_idx++;
  58	}
  59	return nfrags;
  60}
  61
  62static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
  63static void clear_conn_resources(struct chcr_ktls_info *tx_info);
  64/*
  65 * chcr_ktls_save_keys: calculate and save crypto keys.
  66 * @tx_info - driver specific tls info.
  67 * @crypto_info - tls crypto information.
  68 * @direction - TX/RX direction.
  69 * return - SUCCESS/FAILURE.
  70 */
  71static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
  72			       struct tls_crypto_info *crypto_info,
  73			       enum tls_offload_ctx_dir direction)
  74{
  75	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
  76	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
  77	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
  78	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
  79	struct crypto_aes_ctx aes_ctx;
  80	unsigned char *key, *salt;
  81
  82	switch (crypto_info->cipher_type) {
  83	case TLS_CIPHER_AES_GCM_128:
  84		info_128_gcm =
  85			(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
  86		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
  87		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
  88		tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
  89		mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
  90		tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  91		tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
  92
  93		ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
  94		key = info_128_gcm->key;
  95		salt = info_128_gcm->salt;
  96		tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
  97
  98		/* The SCMD fields used when encrypting a full TLS
  99		 * record. Its a one time calculation till the
 100		 * connection exists.
 101		 */
 102		tx_info->scmd0_seqno_numivs =
 103			SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
 104			SCMD_CIPH_AUTH_SEQ_CTRL_F |
 105			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
 106			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
 107			SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
 108			SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
 109			SCMD_NUM_IVS_V(1);
 110
 111		/* keys will be sent inline. */
 112		tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
 113
 114		/* The SCMD fields used when encrypting a partial TLS
 115		 * record (no trailer and possibly a truncated payload).
 116		 */
 117		tx_info->scmd0_short_seqno_numivs =
 118			SCMD_CIPH_AUTH_SEQ_CTRL_F |
 119			SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
 120			SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
 121			SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
 122
 123		tx_info->scmd0_short_ivgen_hdrlen =
 124			tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
 125
 126		break;
 127
 128	default:
 129		pr_err("GCM: cipher type 0x%x not supported\n",
 130		       crypto_info->cipher_type);
 131		ret = -EINVAL;
 132		goto out;
 133	}
 134
 135	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
 136		       roundup(keylen, 16) + ghash_size;
 137	/* Calculate the H = CIPH(K, 0 repeated 16 times).
 138	 * It will go in key context
 139	 */
 140
 141	ret = aes_expandkey(&aes_ctx, key, keylen);
 142	if (ret)
 143		goto out;
 144
 145	memset(ghash_h, 0, ghash_size);
 146	aes_encrypt(&aes_ctx, ghash_h, ghash_h);
 147	memzero_explicit(&aes_ctx, sizeof(aes_ctx));
 148
 149	/* fill the Key context */
 150	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 151		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
 152						 mac_key_size,
 153						 key_ctx_size >> 4);
 154	} else {
 155		ret = -EINVAL;
 156		goto out;
 157	}
 158
 159	memcpy(kctx->salt, salt, tx_info->salt_size);
 160	memcpy(kctx->key, key, keylen);
 161	memcpy(kctx->key + keylen, ghash_h, ghash_size);
 162	tx_info->key_ctx_len = key_ctx_size;
 163
 164out:
 165	return ret;
 166}
 167
 168/*
 169 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
 170 * @sk - tcp socket.
 171 * @tx_info - driver specific tls info.
 172 * @atid - connection active tid.
 173 * return - send success/failure.
 174 */
 175static int chcr_ktls_act_open_req(struct sock *sk,
 176				  struct chcr_ktls_info *tx_info,
 177				  int atid)
 178{
 179	struct inet_sock *inet = inet_sk(sk);
 180	struct cpl_t6_act_open_req *cpl6;
 181	struct cpl_act_open_req *cpl;
 182	struct sk_buff *skb;
 183	unsigned int len;
 184	int qid_atid;
 185	u64 options;
 186
 187	len = sizeof(*cpl6);
 188	skb = alloc_skb(len, GFP_KERNEL);
 189	if (unlikely(!skb))
 190		return -ENOMEM;
 191	/* mark it a control pkt */
 192	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 193
 194	cpl6 = __skb_put_zero(skb, len);
 195	cpl = (struct cpl_act_open_req *)cpl6;
 196	INIT_TP_WR(cpl6, 0);
 197	qid_atid = TID_QID_V(tx_info->rx_qid) |
 198		   TID_TID_V(atid);
 199	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
 200	cpl->local_port = inet->inet_sport;
 201	cpl->peer_port = inet->inet_dport;
 202	cpl->local_ip = inet->inet_rcv_saddr;
 203	cpl->peer_ip = inet->inet_daddr;
 204
 205	/* fill first 64 bit option field. */
 206	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 207		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 208	cpl->opt0 = cpu_to_be64(options);
 209
 210	/* next 64 bit option field. */
 211	options =
 212		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 213	cpl->opt2 = htonl(options);
 214
 215	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 216}
 217
 218#if IS_ENABLED(CONFIG_IPV6)
 219/*
 220 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
 221 * @sk - tcp socket.
 222 * @tx_info - driver specific tls info.
 223 * @atid - connection active tid.
 224 * return - send success/failure.
 225 */
 226static int chcr_ktls_act_open_req6(struct sock *sk,
 227				   struct chcr_ktls_info *tx_info,
 228				   int atid)
 229{
 230	struct inet_sock *inet = inet_sk(sk);
 231	struct cpl_t6_act_open_req6 *cpl6;
 232	struct cpl_act_open_req6 *cpl;
 233	struct sk_buff *skb;
 234	unsigned int len;
 235	int qid_atid;
 236	u64 options;
 237
 238	len = sizeof(*cpl6);
 239	skb = alloc_skb(len, GFP_KERNEL);
 240	if (unlikely(!skb))
 241		return -ENOMEM;
 242	/* mark it a control pkt */
 243	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 244
 245	cpl6 = __skb_put_zero(skb, len);
 246	cpl = (struct cpl_act_open_req6 *)cpl6;
 247	INIT_TP_WR(cpl6, 0);
 248	qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
 249	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
 250	cpl->local_port = inet->inet_sport;
 251	cpl->peer_port = inet->inet_dport;
 252	cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
 253	cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
 254	cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
 255	cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
 256
 257	/* first 64 bit option field. */
 258	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 259		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 260	cpl->opt0 = cpu_to_be64(options);
 261	/* next 64 bit option field. */
 262	options =
 263		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 264	cpl->opt2 = htonl(options);
 265
 266	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 267}
 268#endif /* #if IS_ENABLED(CONFIG_IPV6) */
 269
 270/*
 271 * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
 272 * @sk - tcp socket.
 273 * @tx_info - driver specific tls info.
 274 * return: NET_TX_OK/NET_XMIT_DROP
 275 */
 276static int chcr_setup_connection(struct sock *sk,
 277				 struct chcr_ktls_info *tx_info)
 278{
 279	struct tid_info *t = &tx_info->adap->tids;
 280	int atid, ret = 0;
 281
 282	atid = cxgb4_alloc_atid(t, tx_info);
 283	if (atid == -1)
 284		return -EINVAL;
 285
 286	tx_info->atid = atid;
 287
 288	if (tx_info->ip_family == AF_INET) {
 289		ret = chcr_ktls_act_open_req(sk, tx_info, atid);
 290#if IS_ENABLED(CONFIG_IPV6)
 291	} else {
 292		ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
 293				     &sk->sk_v6_rcv_saddr,
 294				     1);
 295		if (ret)
 296			return ret;
 297		ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
 298#endif
 299	}
 300
 301	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
 302	 * success, if any other return type clear atid and return that failure.
 303	 */
 304	if (ret) {
 305		if (ret == NET_XMIT_CN) {
 306			ret = 0;
 307		} else {
 308#if IS_ENABLED(CONFIG_IPV6)
 309			/* clear clip entry */
 310			if (tx_info->ip_family == AF_INET6)
 311				cxgb4_clip_release(tx_info->netdev,
 312						   (const u32 *)
 313						   &sk->sk_v6_rcv_saddr,
 314						   1);
 315#endif
 316			cxgb4_free_atid(t, atid);
 317		}
 318	}
 319
 320	return ret;
 321}
 322
 323/*
 324 * chcr_set_tcb_field: update tcb fields.
 325 * @tx_info - driver specific tls info.
 326 * @word - TCB word.
 327 * @mask - TCB word related mask.
 328 * @val - TCB word related value.
 329 * @no_reply - set 1 if not looking for TP response.
 330 */
 331static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
 332			      u64 mask, u64 val, int no_reply)
 333{
 334	struct cpl_set_tcb_field *req;
 335	struct sk_buff *skb;
 336
 337	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
 338	if (!skb)
 339		return -ENOMEM;
 340
 341	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
 342	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
 343	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 344				NO_REPLY_V(no_reply));
 345	req->word_cookie = htons(TCB_WORD_V(word));
 346	req->mask = cpu_to_be64(mask);
 347	req->val = cpu_to_be64(val);
 348
 349	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 350	return cxgb4_ofld_send(tx_info->netdev, skb);
 351}
 352
 353/*
 354 * chcr_ktls_dev_del:  call back for tls_dev_del.
 355 * Remove the tid and l2t entry and close the connection.
 356 * it per connection basis.
 357 * @netdev - net device.
 358 * @tls_cts - tls context.
 359 * @direction - TX/RX crypto direction
 360 */
 361static void chcr_ktls_dev_del(struct net_device *netdev,
 362			      struct tls_context *tls_ctx,
 363			      enum tls_offload_ctx_dir direction)
 364{
 365	struct chcr_ktls_info *tx_info = chcr_get_ktls_tx_info(tls_ctx);
 366	struct ch_ktls_port_stats_debug *port_stats;
 367	struct chcr_ktls_uld_ctx *u_ctx;
 368
 369	if (!tx_info)
 370		return;
 371
 372	u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle;
 373	if (u_ctx && u_ctx->detach)
 374		return;
 375	/* clear l2t entry */
 376	if (tx_info->l2te)
 377		cxgb4_l2t_release(tx_info->l2te);
 378
 379#if IS_ENABLED(CONFIG_IPV6)
 380	/* clear clip entry */
 381	if (tx_info->ip_family == AF_INET6)
 382		cxgb4_clip_release(netdev, (const u32 *)
 383				   &tx_info->sk->sk_v6_rcv_saddr,
 384				   1);
 385#endif
 386
 387	/* clear tid */
 388	if (tx_info->tid != -1) {
 389		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 390				 tx_info->tid, tx_info->ip_family);
 391
 392		xa_erase(&u_ctx->tid_list, tx_info->tid);
 393	}
 394
 395	port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
 396	atomic64_inc(&port_stats->ktls_tx_connection_close);
 397	kvfree(tx_info);
 398	chcr_set_ktls_tx_info(tls_ctx, NULL);
 399	/* release module refcount */
 400	module_put(THIS_MODULE);
 401}
 402
 403/*
 404 * chcr_ktls_dev_add:  call back for tls_dev_add.
 405 * Create a tcb entry for TP. Also add l2t entry for the connection. And
 406 * generate keys & save those keys locally.
 407 * @netdev - net device.
 408 * @tls_cts - tls context.
 409 * @direction - TX/RX crypto direction
 410 * return: SUCCESS/FAILURE.
 411 */
 412static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
 413			     enum tls_offload_ctx_dir direction,
 414			     struct tls_crypto_info *crypto_info,
 415			     u32 start_offload_tcp_sn)
 416{
 417	struct tls_context *tls_ctx = tls_get_ctx(sk);
 418	struct ch_ktls_port_stats_debug *port_stats;
 419	struct chcr_ktls_uld_ctx *u_ctx;
 420	struct chcr_ktls_info *tx_info;
 421	struct dst_entry *dst;
 422	struct adapter *adap;
 423	struct port_info *pi;
 424	struct neighbour *n;
 425	u8 daaddr[16];
 426	int ret = -1;
 427
 428	pi = netdev_priv(netdev);
 429	adap = pi->adapter;
 430	port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
 431	atomic64_inc(&port_stats->ktls_tx_connection_open);
 432	u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
 433
 434	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
 435		pr_err("not expecting for RX direction\n");
 436		goto out;
 437	}
 438
 439	if (chcr_get_ktls_tx_info(tls_ctx))
 440		goto out;
 441
 442	if (u_ctx && u_ctx->detach)
 443		goto out;
 444
 445	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
 446	if (!tx_info)
 447		goto out;
 448
 449	tx_info->sk = sk;
 450	spin_lock_init(&tx_info->lock);
 451	/* initialize tid and atid to -1, 0 is a also a valid id. */
 452	tx_info->tid = -1;
 453	tx_info->atid = -1;
 454
 455	tx_info->adap = adap;
 456	tx_info->netdev = netdev;
 457	tx_info->first_qset = pi->first_qset;
 458	tx_info->tx_chan = pi->tx_chan;
 459	tx_info->smt_idx = pi->smt_idx;
 460	tx_info->port_id = pi->port_id;
 461	tx_info->prev_ack = 0;
 462	tx_info->prev_win = 0;
 463
 464	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
 465	if (unlikely(tx_info->rx_qid < 0))
 466		goto free_tx_info;
 467
 468	tx_info->prev_seq = start_offload_tcp_sn;
 469	tx_info->tcp_start_seq_number = start_offload_tcp_sn;
 470
 471	/* save crypto keys */
 472	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
 473	if (ret < 0)
 474		goto free_tx_info;
 475
 476	/* get peer ip */
 477	if (sk->sk_family == AF_INET) {
 478		memcpy(daaddr, &sk->sk_daddr, 4);
 479		tx_info->ip_family = AF_INET;
 480#if IS_ENABLED(CONFIG_IPV6)
 481	} else {
 482		if (!ipv6_only_sock(sk) &&
 483		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
 484			memcpy(daaddr, &sk->sk_daddr, 4);
 485			tx_info->ip_family = AF_INET;
 486		} else {
 487			memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
 488			tx_info->ip_family = AF_INET6;
 489		}
 490#endif
 491	}
 492
 493	/* get the l2t index */
 494	dst = sk_dst_get(sk);
 495	if (!dst) {
 496		pr_err("DST entry not found\n");
 497		goto free_tx_info;
 498	}
 499	n = dst_neigh_lookup(dst, daaddr);
 500	if (!n || !n->dev) {
 501		pr_err("neighbour not found\n");
 502		dst_release(dst);
 503		goto free_tx_info;
 504	}
 505	tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
 506
 507	neigh_release(n);
 508	dst_release(dst);
 509
 510	if (!tx_info->l2te) {
 511		pr_err("l2t entry not found\n");
 512		goto free_tx_info;
 513	}
 514
 515	/* Driver shouldn't be removed until any single connection exists */
 516	if (!try_module_get(THIS_MODULE))
 517		goto free_l2t;
 518
 519	init_completion(&tx_info->completion);
 520	/* create a filter and call cxgb4_l2t_send to send the packet out, which
 521	 * will take care of updating l2t entry in hw if not already done.
 522	 */
 523	tx_info->open_state = CH_KTLS_OPEN_PENDING;
 524
 525	if (chcr_setup_connection(sk, tx_info))
 526		goto put_module;
 527
 528	/* Wait for reply */
 529	wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
 530	spin_lock_bh(&tx_info->lock);
 531	if (tx_info->open_state) {
 532		/* need to wait for hw response, can't free tx_info yet. */
 533		if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
 534			tx_info->pending_close = true;
 535		else
 536			spin_unlock_bh(&tx_info->lock);
 537		/* if in pending close, free the lock after the cleanup */
 538		goto put_module;
 539	}
 540	spin_unlock_bh(&tx_info->lock);
 541
 542	/* initialize tcb */
 543	reinit_completion(&tx_info->completion);
 544	/* mark it pending for hw response */
 545	tx_info->open_state = CH_KTLS_OPEN_PENDING;
 546
 547	if (chcr_init_tcb_fields(tx_info))
 548		goto free_tid;
 549
 550	/* Wait for reply */
 551	wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
 552	spin_lock_bh(&tx_info->lock);
 553	if (tx_info->open_state) {
 554		/* need to wait for hw response, can't free tx_info yet. */
 555		tx_info->pending_close = true;
 556		/* free the lock after cleanup */
 557		goto free_tid;
 558	}
 559	spin_unlock_bh(&tx_info->lock);
 560
 561	if (!cxgb4_check_l2t_valid(tx_info->l2te))
 562		goto free_tid;
 563
 564	atomic64_inc(&port_stats->ktls_tx_ctx);
 565	chcr_set_ktls_tx_info(tls_ctx, tx_info);
 566
 567	return 0;
 568
 569free_tid:
 570#if IS_ENABLED(CONFIG_IPV6)
 571	/* clear clip entry */
 572	if (tx_info->ip_family == AF_INET6)
 573		cxgb4_clip_release(netdev, (const u32 *)
 574				   &sk->sk_v6_rcv_saddr,
 575				   1);
 576#endif
 577	cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 578			 tx_info->tid, tx_info->ip_family);
 579
 580	xa_erase(&u_ctx->tid_list, tx_info->tid);
 581
 582put_module:
 583	/* release module refcount */
 584	module_put(THIS_MODULE);
 585free_l2t:
 586	cxgb4_l2t_release(tx_info->l2te);
 587free_tx_info:
 588	if (tx_info->pending_close)
 589		spin_unlock_bh(&tx_info->lock);
 590	else
 591		kvfree(tx_info);
 592out:
 593	atomic64_inc(&port_stats->ktls_tx_connection_fail);
 594	return -1;
 595}
 596
 597/*
 598 * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
 599 *			  handling.
 600 * @tx_info - driver specific tls info.
 601 * return: NET_TX_OK/NET_XMIT_DROP
 602 */
 603static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
 604{
 605	int  ret = 0;
 606
 607	/* set tcb in offload and bypass */
 608	ret =
 609	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
 610			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
 611			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
 612	if (ret)
 613		return ret;
 614	/* reset snd_una and snd_next fields in tcb */
 615	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
 616				 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
 617				 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
 618				 0, 1);
 619	if (ret)
 620		return ret;
 621
 622	/* reset send max */
 623	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
 624				 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
 625				 0, 1);
 626	if (ret)
 627		return ret;
 628
 629	/* update l2t index and request for tp reply to confirm tcb is
 630	 * initialised to handle tx traffic.
 631	 */
 632	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
 633				 TCB_L2T_IX_V(TCB_L2T_IX_M),
 634				 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
 635	return ret;
 636}
 637
 638/*
 639 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
 640 */
 641static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
 642				      unsigned char *input)
 643{
 644	const struct cpl_act_open_rpl *p = (void *)input;
 645	struct chcr_ktls_info *tx_info = NULL;
 646	struct tls_offload_context_tx *tx_ctx;
 647	struct chcr_ktls_uld_ctx *u_ctx;
 648	unsigned int atid, tid, status;
 649	struct tls_context *tls_ctx;
 650	struct tid_info *t;
 651	int ret = 0;
 652
 653	tid = GET_TID(p);
 654	status = AOPEN_STATUS_G(ntohl(p->atid_status));
 655	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
 656
 657	t = &adap->tids;
 658	tx_info = lookup_atid(t, atid);
 659
 660	if (!tx_info || tx_info->atid != atid) {
 661		pr_err("%s: incorrect tx_info or atid\n", __func__);
 662		return -1;
 663	}
 664
 665	cxgb4_free_atid(t, atid);
 666	tx_info->atid = -1;
 667
 668	spin_lock(&tx_info->lock);
 669	/* HW response is very close, finish pending cleanup */
 670	if (tx_info->pending_close) {
 671		spin_unlock(&tx_info->lock);
 672		if (!status) {
 673			cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 674					 tid, tx_info->ip_family);
 675		}
 676		kvfree(tx_info);
 677		return 0;
 678	}
 679
 680	if (!status) {
 681		tx_info->tid = tid;
 682		cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
 683		/* Adding tid */
 684		tls_ctx = tls_get_ctx(tx_info->sk);
 685		tx_ctx = tls_offload_ctx_tx(tls_ctx);
 686		u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
 687		if (u_ctx) {
 688			ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx,
 689					   GFP_NOWAIT);
 690			if (ret < 0) {
 691				pr_err("%s: Failed to allocate tid XA entry = %d\n",
 692				       __func__, tx_info->tid);
 693				tx_info->open_state = CH_KTLS_OPEN_FAILURE;
 694				goto out;
 695			}
 696		}
 697		tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
 698	} else {
 699		tx_info->open_state = CH_KTLS_OPEN_FAILURE;
 700	}
 701out:
 702	spin_unlock(&tx_info->lock);
 703
 704	complete(&tx_info->completion);
 705	return ret;
 706}
 707
 708/*
 709 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
 710 */
 711static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
 712{
 713	const struct cpl_set_tcb_rpl *p = (void *)input;
 714	struct chcr_ktls_info *tx_info = NULL;
 715	struct tid_info *t;
 716	u32 tid;
 717
 718	tid = GET_TID(p);
 719
 720	t = &adap->tids;
 721	tx_info = lookup_tid(t, tid);
 722
 723	if (!tx_info || tx_info->tid != tid) {
 724		pr_err("%s: incorrect tx_info or tid\n", __func__);
 725		return -1;
 726	}
 727
 728	spin_lock(&tx_info->lock);
 729	if (tx_info->pending_close) {
 730		spin_unlock(&tx_info->lock);
 731		kvfree(tx_info);
 732		return 0;
 733	}
 734	tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
 735	spin_unlock(&tx_info->lock);
 736
 737	complete(&tx_info->completion);
 738	return 0;
 739}
 740
 741static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 742					u32 tid, void *pos, u16 word,
 743					struct sge_eth_txq *q, u64 mask,
 744					u64 val, u32 reply)
 745{
 746	struct cpl_set_tcb_field_core *cpl;
 747	struct ulptx_idata *idata;
 748	struct ulp_txpkt *txpkt;
 749
 750	/* ULP_TXPKT */
 751	txpkt = pos;
 752	txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
 753				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
 754				ULP_TXPKT_FID_V(q->q.cntxt_id) |
 755				ULP_TXPKT_RO_F);
 756	txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
 757
 758	/* ULPTX_IDATA sub-command */
 759	idata = (struct ulptx_idata *)(txpkt + 1);
 760	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
 761	idata->len = htonl(sizeof(*cpl));
 762	pos = idata + 1;
 763
 764	cpl = pos;
 765	/* CPL_SET_TCB_FIELD */
 766	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 767	cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 768			NO_REPLY_V(!reply));
 769	cpl->word_cookie = htons(TCB_WORD_V(word));
 770	cpl->mask = cpu_to_be64(mask);
 771	cpl->val = cpu_to_be64(val);
 772
 773	/* ULPTX_NOOP */
 774	idata = (struct ulptx_idata *)(cpl + 1);
 775	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
 776	idata->len = htonl(0);
 777	pos = idata + 1;
 778
 779	return pos;
 780}
 781
 782
 783/*
 784 * chcr_write_cpl_set_tcb_ulp: update tcb values.
 785 * TCB is responsible to create tcp headers, so all the related values
 786 * should be correctly updated.
 787 * @tx_info - driver specific tls info.
 788 * @q - tx queue on which packet is going out.
 789 * @tid - TCB identifier.
 790 * @pos - current index where should we start writing.
 791 * @word - TCB word.
 792 * @mask - TCB word related mask.
 793 * @val - TCB word related value.
 794 * @reply - set 1 if looking for TP response.
 795 * return - next position to write.
 796 */
 797static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 798					struct sge_eth_txq *q, u32 tid,
 799					void *pos, u16 word, u64 mask,
 800					u64 val, u32 reply)
 801{
 802	int left = (void *)q->q.stat - pos;
 803
 804	if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
 805		if (!left) {
 806			pos = q->q.desc;
 807		} else {
 808			u8 buf[48] = {0};
 809
 810			__chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
 811						     mask, val, reply);
 812
 813			return chcr_copy_to_txd(buf, &q->q, pos,
 814						CHCR_SET_TCB_FIELD_LEN);
 815		}
 816	}
 817
 818	pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
 819					   mask, val, reply);
 820
 821	/* check again if we are at the end of the queue */
 822	if (left == CHCR_SET_TCB_FIELD_LEN)
 823		pos = q->q.desc;
 824
 825	return pos;
 826}
 827
 828/*
 829 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
 830 * with updated values like tcp seq, ack, window etc.
 831 * @tx_info - driver specific tls info.
 832 * @q - TX queue.
 833 * @tcp_seq
 834 * @tcp_ack
 835 * @tcp_win
 836 * return: NETDEV_TX_BUSY/NET_TX_OK.
 837 */
 838static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
 839				   struct sge_eth_txq *q, u64 tcp_seq,
 840				   u64 tcp_ack, u64 tcp_win, bool offset)
 841{
 842	bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
 843	struct ch_ktls_port_stats_debug *port_stats;
 844	u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
 845	struct fw_ulptx_wr *wr;
 846	int credits;
 847	void *pos;
 848
 849	wr_len = sizeof(*wr);
 850	/* there can be max 4 cpls, check if we have enough credits */
 851	len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 852	ndesc = DIV_ROUND_UP(len, 64);
 853
 854	credits = chcr_txq_avail(&q->q) - ndesc;
 855	if (unlikely(credits < 0)) {
 856		chcr_eth_txq_stop(q);
 857		return NETDEV_TX_BUSY;
 858	}
 859
 860	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
 861		chcr_eth_txq_stop(q);
 862		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
 863	}
 864
 865	pos = &q->q.desc[q->q.pidx];
 866	/* make space for WR, we'll fill it later when we know all the cpls
 867	 * being sent out and have complete length.
 868	 */
 869	wr = pos;
 870	pos += wr_len;
 871	/* update tx_max if its a re-transmit or the first wr */
 872	if (first_wr || tcp_seq != tx_info->prev_seq) {
 873		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 874						 TCB_TX_MAX_W,
 875						 TCB_TX_MAX_V(TCB_TX_MAX_M),
 876						 TCB_TX_MAX_V(tcp_seq), 0);
 877		cpl++;
 878	}
 879	/* reset snd una if it's a re-transmit pkt */
 880	if (tcp_seq != tx_info->prev_seq || offset) {
 881		/* reset snd_una */
 882		port_stats =
 883			&tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
 884		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 885						 TCB_SND_UNA_RAW_W,
 886						 TCB_SND_UNA_RAW_V
 887						 (TCB_SND_UNA_RAW_M),
 888						 TCB_SND_UNA_RAW_V(0), 0);
 889		if (tcp_seq != tx_info->prev_seq)
 890			atomic64_inc(&port_stats->ktls_tx_ooo);
 891		cpl++;
 892	}
 893	/* update ack */
 894	if (first_wr || tx_info->prev_ack != tcp_ack) {
 895		pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 896						 TCB_RCV_NXT_W,
 897						 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
 898						 TCB_RCV_NXT_V(tcp_ack), 0);
 899		tx_info->prev_ack = tcp_ack;
 900		cpl++;
 901	}
 902	/* update receive window */
 903	if (first_wr || tx_info->prev_win != tcp_win) {
 904		chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 905					   TCB_RCV_WND_W,
 906					   TCB_RCV_WND_V(TCB_RCV_WND_M),
 907					   TCB_RCV_WND_V(tcp_win), 0);
 908		tx_info->prev_win = tcp_win;
 909		cpl++;
 910	}
 911
 912	if (cpl) {
 913		/* get the actual length */
 914		len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 915		/* ULPTX wr */
 916		wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
 917		wr->cookie = 0;
 918		/* fill len in wr field */
 919		wr->flowid_len16 = htonl(wr_mid |
 920					 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
 921
 922		ndesc = DIV_ROUND_UP(len, 64);
 923		chcr_txq_advance(&q->q, ndesc);
 924		cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
 925	}
 926	return 0;
 927}
 928
 929/*
 930 * chcr_ktls_get_tx_flits
 931 * returns number of flits to be sent out, it includes key context length, WR
 932 * size and skb fragments.
 933 */
 934static unsigned int
 935chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
 936{
 937	return chcr_sgl_len(nr_frags) +
 938	       DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
 939}
 940
 941/*
 942 * chcr_ktls_check_tcp_options: To check if there is any TCP option available
 943 * other than timestamp.
 944 * @skb - skb contains partial record..
 945 * return: 1 / 0
 946 */
 947static int
 948chcr_ktls_check_tcp_options(struct tcphdr *tcp)
 949{
 950	int cnt, opt, optlen;
 951	u_char *cp;
 952
 953	cp = (u_char *)(tcp + 1);
 954	cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
 955	for (; cnt > 0; cnt -= optlen, cp += optlen) {
 956		opt = cp[0];
 957		if (opt == TCPOPT_EOL)
 958			break;
 959		if (opt == TCPOPT_NOP) {
 960			optlen = 1;
 961		} else {
 962			if (cnt < 2)
 963				break;
 964			optlen = cp[1];
 965			if (optlen < 2 || optlen > cnt)
 966				break;
 967		}
 968		switch (opt) {
 969		case TCPOPT_NOP:
 970			break;
 971		default:
 972			return 1;
 973		}
 974	}
 975	return 0;
 976}
 977
 978/*
 979 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
 980 * send out separately.
 981 * @tx_info - driver specific tls info.
 982 * @skb - skb contains partial record..
 983 * @q - TX queue.
 984 * @tx_chan - channel number.
 985 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
 986 */
 987static int
 988chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
 989			    struct sge_eth_txq *q, uint32_t tx_chan)
 990{
 991	struct fw_eth_tx_pkt_wr *wr;
 992	struct cpl_tx_pkt_core *cpl;
 993	u32 ctrl, iplen, maclen;
 994	struct ipv6hdr *ip6;
 995	unsigned int ndesc;
 996	struct tcphdr *tcp;
 997	int len16, pktlen;
 998	struct iphdr *ip;
 999	u32 wr_mid = 0;
1000	int credits;
1001	u8 buf[150];
1002	u64 cntrl1;
1003	void *pos;
1004
1005	iplen = skb_network_header_len(skb);
1006	maclen = skb_mac_header_len(skb);
1007
1008	/* packet length = eth hdr len + ip hdr len + tcp hdr len
1009	 * (including options).
1010	 */
1011	pktlen = skb_tcp_all_headers(skb);
1012
1013	ctrl = sizeof(*cpl) + pktlen;
1014	len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1015	/* check how many descriptors needed */
1016	ndesc = DIV_ROUND_UP(len16, 4);
1017
1018	credits = chcr_txq_avail(&q->q) - ndesc;
1019	if (unlikely(credits < 0)) {
1020		chcr_eth_txq_stop(q);
1021		return NETDEV_TX_BUSY;
1022	}
1023
1024	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1025		chcr_eth_txq_stop(q);
1026		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1027	}
1028
1029	pos = &q->q.desc[q->q.pidx];
1030	wr = pos;
1031
1032	/* Firmware work request header */
1033	wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1034			       FW_WR_IMMDLEN_V(ctrl));
1035
1036	wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1037	wr->r3 = 0;
1038
1039	cpl = (void *)(wr + 1);
1040
1041	/* CPL header */
1042	cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1043			   TXPKT_PF_V(tx_info->adap->pf));
1044	cpl->pack = 0;
1045	cpl->len = htons(pktlen);
1046
1047	memcpy(buf, skb->data, pktlen);
1048	if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1049		/* we need to correct ip header len */
1050		ip = (struct iphdr *)(buf + maclen);
1051		ip->tot_len = htons(pktlen - maclen);
1052		cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1053	} else {
1054		ip6 = (struct ipv6hdr *)(buf + maclen);
1055		ip6->payload_len = htons(pktlen - maclen - iplen);
1056		cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1057	}
1058
1059	cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1060		  TXPKT_IPHDR_LEN_V(iplen);
1061	/* checksum offload */
1062	cpl->ctrl1 = cpu_to_be64(cntrl1);
1063
1064	pos = cpl + 1;
1065
1066	/* now take care of the tcp header, if fin is not set then clear push
1067	 * bit as well, and if fin is set, it will be sent at the last so we
1068	 * need to update the tcp sequence number as per the last packet.
1069	 */
1070	tcp = (struct tcphdr *)(buf + maclen + iplen);
1071
1072	if (!tcp->fin)
1073		tcp->psh = 0;
1074	else
1075		tcp->seq = htonl(tx_info->prev_seq);
1076
1077	chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1078
1079	chcr_txq_advance(&q->q, ndesc);
1080	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1081	return 0;
1082}
1083
1084/*
1085 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1086 * received has partial end part of the record, send out the complete record, so
1087 * that crypto block will be able to generate TAG/HASH.
1088 * @skb - segment which has complete or partial end part.
1089 * @tx_info - driver specific tls info.
1090 * @q - TX queue.
1091 * @tcp_seq
1092 * @tcp_push - tcp push bit.
1093 * @mss - segment size.
1094 * return: NETDEV_TX_BUSY/NET_TX_OK.
1095 */
1096static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1097				      struct chcr_ktls_info *tx_info,
1098				      struct sge_eth_txq *q, u32 tcp_seq,
1099				      bool is_last_wr, u32 data_len,
1100				      u32 skb_offset, u32 nfrags,
1101				      bool tcp_push, u32 mss)
1102{
1103	u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1104	struct adapter *adap = tx_info->adap;
1105	int credits, left, last_desc;
1106	struct tx_sw_desc *sgl_sdesc;
1107	struct cpl_tx_data *tx_data;
1108	struct cpl_tx_sec_pdu *cpl;
1109	struct ulptx_idata *idata;
1110	struct ulp_txpkt *ulptx;
1111	struct fw_ulptx_wr *wr;
1112	void *pos;
1113	u64 *end;
1114
1115	/* get the number of flits required */
1116	flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1117	/* number of descriptors */
1118	ndesc = chcr_flits_to_desc(flits);
1119	/* check if enough credits available */
1120	credits = chcr_txq_avail(&q->q) - ndesc;
1121	if (unlikely(credits < 0)) {
1122		chcr_eth_txq_stop(q);
1123		return NETDEV_TX_BUSY;
1124	}
1125
1126	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1127		/* Credits are below the threshold values, stop the queue after
1128		 * injecting the Work Request for this packet.
1129		 */
1130		chcr_eth_txq_stop(q);
1131		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1132	}
1133
1134	last_desc = q->q.pidx + ndesc - 1;
1135	if (last_desc >= q->q.size)
1136		last_desc -= q->q.size;
1137	sgl_sdesc = &q->q.sdesc[last_desc];
1138
1139	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1140		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1141		q->mapping_err++;
1142		return NETDEV_TX_BUSY;
1143	}
1144
1145	if (!is_last_wr)
1146		skb_get(skb);
1147
1148	pos = &q->q.desc[q->q.pidx];
1149	end = (u64 *)pos + flits;
1150	/* FW_ULPTX_WR */
1151	wr = pos;
1152	/* WR will need len16 */
1153	len16 = DIV_ROUND_UP(flits, 2);
1154	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1155	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1156	wr->cookie = 0;
1157	pos += sizeof(*wr);
1158	/* ULP_TXPKT */
1159	ulptx = pos;
1160	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1161				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1162				ULP_TXPKT_FID_V(q->q.cntxt_id) |
1163				ULP_TXPKT_RO_F);
1164	ulptx->len = htonl(len16 - 1);
1165	/* ULPTX_IDATA sub-command */
1166	idata = (struct ulptx_idata *)(ulptx + 1);
1167	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1168	/* idata length will include cpl_tx_sec_pdu + key context size +
1169	 * cpl_tx_data header.
1170	 */
1171	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1172			   sizeof(*tx_data));
1173	/* SEC CPL */
1174	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1175	cpl->op_ivinsrtofst =
1176		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1177		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1178		      CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1179		      CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1180	cpl->pldlen = htonl(data_len);
1181
1182	/* encryption should start after tls header size + iv size */
1183	cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1184
1185	cpl->aadstart_cipherstop_hi =
1186		htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1187		      CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1188		      CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1189
1190	/* authentication will also start after tls header + iv size */
1191	cpl->cipherstop_lo_authinsert =
1192	htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1193	      CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1194	      CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1195
1196	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1197	cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1198	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1199	cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1200
1201	pos = cpl + 1;
1202	/* check if space left to fill the keys */
1203	left = (void *)q->q.stat - pos;
1204	if (!left) {
1205		left = (void *)end - (void *)q->q.stat;
1206		pos = q->q.desc;
1207		end = pos + left;
1208	}
1209
1210	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1211			       tx_info->key_ctx_len);
1212	left = (void *)q->q.stat - pos;
1213
1214	if (!left) {
1215		left = (void *)end - (void *)q->q.stat;
1216		pos = q->q.desc;
1217		end = pos + left;
1218	}
1219	/* CPL_TX_DATA */
1220	tx_data = (void *)pos;
1221	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1222	tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1223
1224	tx_data->rsvd = htonl(tcp_seq);
1225
1226	tx_data->flags = htonl(TX_BYPASS_F);
1227	if (tcp_push)
1228		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1229
1230	/* check left again, it might go beyond queue limit */
1231	pos = tx_data + 1;
1232	left = (void *)q->q.stat - pos;
1233
1234	/* check the position again */
1235	if (!left) {
1236		left = (void *)end - (void *)q->q.stat;
1237		pos = q->q.desc;
1238		end = pos + left;
1239	}
1240
1241	/* send the complete packet except the header */
1242	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1243				skb_offset, data_len);
1244	sgl_sdesc->skb = skb;
1245
1246	chcr_txq_advance(&q->q, ndesc);
1247	cxgb4_ring_tx_db(adap, &q->q, ndesc);
1248	atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1249
1250	return 0;
1251}
1252
1253/*
1254 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1255 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1256 * and then only send it out.
1257 *
1258 * @skb - skb contains partial record..
1259 * @tx_info - driver specific tls info.
1260 * @q - TX queue.
1261 * @tcp_seq
1262 * @tcp_push - tcp push bit.
1263 * @mss - segment size.
1264 * @tls_rec_offset - offset from start of the tls record.
1265 * @perior_data - data before the current segment, required to make this record
1266 *		  16 byte aligned.
1267 * @prior_data_len - prior_data length (less than 16)
1268 * return: NETDEV_TX_BUSY/NET_TX_OK.
1269 */
1270static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1271				   struct chcr_ktls_info *tx_info,
1272				   struct sge_eth_txq *q,
1273				   u32 tcp_seq, bool tcp_push, u32 mss,
1274				   u32 tls_rec_offset, u8 *prior_data,
1275				   u32 prior_data_len, u32 data_len,
1276				   u32 skb_offset)
1277{
1278	u32 len16, wr_mid = 0, cipher_start, nfrags;
1279	struct adapter *adap = tx_info->adap;
1280	unsigned int flits = 0, ndesc;
1281	int credits, left, last_desc;
1282	struct tx_sw_desc *sgl_sdesc;
1283	struct cpl_tx_data *tx_data;
1284	struct cpl_tx_sec_pdu *cpl;
1285	struct ulptx_idata *idata;
1286	struct ulp_txpkt *ulptx;
1287	struct fw_ulptx_wr *wr;
1288	__be64 iv_record;
1289	void *pos;
1290	u64 *end;
1291
1292	nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1293	/* get the number of flits required, it's a partial record so 2 flits
1294	 * (AES_BLOCK_SIZE) will be added.
1295	 */
1296	flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1297	/* get the correct 8 byte IV of this record */
1298	iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1299	/* If it's a middle record and not 16 byte aligned to run AES CTR, need
1300	 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1301	 * data will be added.
1302	 */
1303	if (prior_data_len)
1304		flits += 2;
1305	/* number of descriptors */
1306	ndesc = chcr_flits_to_desc(flits);
1307	/* check if enough credits available */
1308	credits = chcr_txq_avail(&q->q) - ndesc;
1309	if (unlikely(credits < 0)) {
1310		chcr_eth_txq_stop(q);
1311		return NETDEV_TX_BUSY;
1312	}
1313
1314	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1315		chcr_eth_txq_stop(q);
1316		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1317	}
1318
1319	last_desc = q->q.pidx + ndesc - 1;
1320	if (last_desc >= q->q.size)
1321		last_desc -= q->q.size;
1322	sgl_sdesc = &q->q.sdesc[last_desc];
1323
1324	if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1325		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1326		q->mapping_err++;
1327		return NETDEV_TX_BUSY;
1328	}
1329
1330	pos = &q->q.desc[q->q.pidx];
1331	end = (u64 *)pos + flits;
1332	/* FW_ULPTX_WR */
1333	wr = pos;
1334	/* WR will need len16 */
1335	len16 = DIV_ROUND_UP(flits, 2);
1336	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1337	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1338	wr->cookie = 0;
1339	pos += sizeof(*wr);
1340	/* ULP_TXPKT */
1341	ulptx = pos;
1342	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1343				ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1344				ULP_TXPKT_FID_V(q->q.cntxt_id) |
1345				ULP_TXPKT_RO_F);
1346	ulptx->len = htonl(len16 - 1);
1347	/* ULPTX_IDATA sub-command */
1348	idata = (struct ulptx_idata *)(ulptx + 1);
1349	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1350	/* idata length will include cpl_tx_sec_pdu + key context size +
1351	 * cpl_tx_data header.
1352	 */
1353	idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1354			   sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1355	/* SEC CPL */
1356	cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1357	/* cipher start will have tls header + iv size extra if its a header
1358	 * part of tls record. else only 16 byte IV will be added.
1359	 */
1360	cipher_start =
1361		AES_BLOCK_LEN + 1 +
1362		(!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1363
1364	cpl->op_ivinsrtofst =
1365		htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1366		      CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1367		      CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1368	cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1369	cpl->aadstart_cipherstop_hi =
1370		htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1371	cpl->cipherstop_lo_authinsert = 0;
1372	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1373	cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1374	cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1375	cpl->scmd1 = 0;
1376
1377	pos = cpl + 1;
1378	/* check if space left to fill the keys */
1379	left = (void *)q->q.stat - pos;
1380	if (!left) {
1381		left = (void *)end - (void *)q->q.stat;
1382		pos = q->q.desc;
1383		end = pos + left;
1384	}
1385
1386	pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1387			       tx_info->key_ctx_len);
1388	left = (void *)q->q.stat - pos;
1389
1390	if (!left) {
1391		left = (void *)end - (void *)q->q.stat;
1392		pos = q->q.desc;
1393		end = pos + left;
1394	}
1395	/* CPL_TX_DATA */
1396	tx_data = (void *)pos;
1397	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1398	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1399			     TX_LENGTH_V(data_len + prior_data_len));
1400	tx_data->rsvd = htonl(tcp_seq);
1401	tx_data->flags = htonl(TX_BYPASS_F);
1402	if (tcp_push)
1403		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1404
1405	/* check left again, it might go beyond queue limit */
1406	pos = tx_data + 1;
1407	left = (void *)q->q.stat - pos;
1408
1409	/* check the position again */
1410	if (!left) {
1411		left = (void *)end - (void *)q->q.stat;
1412		pos = q->q.desc;
1413		end = pos + left;
1414	}
1415	/* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1416	 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1417	 */
1418	memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1419	memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1420	*(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1421		htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1422		(TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1423
1424	pos += 16;
1425	/* Prior_data_len will always be less than 16 bytes, fill the
1426	 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1427	 * to 0.
1428	 */
1429	if (prior_data_len)
1430		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1431	/* send the complete packet except the header */
1432	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1433				skb_offset, data_len);
1434	sgl_sdesc->skb = skb;
1435
1436	chcr_txq_advance(&q->q, ndesc);
1437	cxgb4_ring_tx_db(adap, &q->q, ndesc);
1438
1439	return 0;
1440}
1441
1442/*
1443 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1444 * only plain text (only tls header and iv)
1445 * @tx_info - driver specific tls info.
1446 * @skb - skb contains partial record..
1447 * @tcp_seq
1448 * @mss - segment size.
1449 * @tcp_push - tcp push bit.
1450 * @q - TX queue.
1451 * @port_id : port number
1452 * @perior_data - data before the current segment, required to make this record
1453 *		 16 byte aligned.
1454 * @prior_data_len - prior_data length (less than 16)
1455 * return: NETDEV_TX_BUSY/NET_TX_OK.
1456 */
1457static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1458				 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1459				 bool tcp_push, struct sge_eth_txq *q,
1460				 u32 port_id, u8 *prior_data,
1461				 u32 data_len, u32 skb_offset,
1462				 u32 prior_data_len)
1463{
1464	int credits, left, len16, last_desc;
1465	unsigned int flits = 0, ndesc;
1466	struct tx_sw_desc *sgl_sdesc;
1467	struct cpl_tx_data *tx_data;
1468	struct ulptx_idata *idata;
1469	struct ulp_txpkt *ulptx;
1470	struct fw_ulptx_wr *wr;
1471	u32 wr_mid = 0, nfrags;
1472	void *pos;
1473	u64 *end;
1474
1475	flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1476	nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1477	flits += chcr_sgl_len(nfrags);
1478	if (prior_data_len)
1479		flits += 2;
1480
1481	/* WR will need len16 */
1482	len16 = DIV_ROUND_UP(flits, 2);
1483	/* check how many descriptors needed */
1484	ndesc = DIV_ROUND_UP(flits, 8);
1485
1486	credits = chcr_txq_avail(&q->q) - ndesc;
1487	if (unlikely(credits < 0)) {
1488		chcr_eth_txq_stop(q);
1489		return NETDEV_TX_BUSY;
1490	}
1491
1492	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1493		chcr_eth_txq_stop(q);
1494		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1495	}
1496
1497	last_desc = q->q.pidx + ndesc - 1;
1498	if (last_desc >= q->q.size)
1499		last_desc -= q->q.size;
1500	sgl_sdesc = &q->q.sdesc[last_desc];
1501
1502	if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1503				   sgl_sdesc->addr) < 0)) {
1504		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1505		q->mapping_err++;
1506		return NETDEV_TX_BUSY;
1507	}
1508
1509	pos = &q->q.desc[q->q.pidx];
1510	end = (u64 *)pos + flits;
1511	/* FW_ULPTX_WR */
1512	wr = pos;
1513	wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1514	wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1515	wr->cookie = 0;
1516	/* ULP_TXPKT */
1517	ulptx = (struct ulp_txpkt *)(wr + 1);
1518	ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1519			ULP_TXPKT_DATAMODIFY_V(0) |
1520			ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1521			ULP_TXPKT_DEST_V(0) |
1522			ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1523	ulptx->len = htonl(len16 - 1);
1524	/* ULPTX_IDATA sub-command */
1525	idata = (struct ulptx_idata *)(ulptx + 1);
1526	idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1527	idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1528	/* CPL_TX_DATA */
1529	tx_data = (struct cpl_tx_data *)(idata + 1);
1530	OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1531	tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1532			     TX_LENGTH_V(data_len + prior_data_len));
1533	/* set tcp seq number */
1534	tx_data->rsvd = htonl(tcp_seq);
1535	tx_data->flags = htonl(TX_BYPASS_F);
1536	if (tcp_push)
1537		tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1538
1539	pos = tx_data + 1;
1540	/* apart from prior_data_len, we should set remaining part of 16 bytes
1541	 * to be zero.
1542	 */
1543	if (prior_data_len)
1544		pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1545
1546	/* check left again, it might go beyond queue limit */
1547	left = (void *)q->q.stat - pos;
1548
1549	/* check the position again */
1550	if (!left) {
1551		left = (void *)end - (void *)q->q.stat;
1552		pos = q->q.desc;
1553		end = pos + left;
1554	}
1555	/* send the complete packet including the header */
1556	cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1557				skb_offset, data_len);
1558	sgl_sdesc->skb = skb;
1559
1560	chcr_txq_advance(&q->q, ndesc);
1561	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1562	return 0;
1563}
1564
1565static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1566				struct sk_buff *skb,
1567				struct sge_eth_txq *q)
1568{
1569	u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1570	struct tx_sw_desc *sgl_sdesc;
1571	struct fw_eth_tx_pkt_wr *wr;
1572	struct cpl_tx_pkt_core *cpl;
1573	unsigned int flits, ndesc;
1574	int credits, last_desc;
1575	u64 cntrl1, *end;
1576	void *pos;
1577
1578	ctrl = sizeof(*cpl);
1579	flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1580
1581	flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1582	len16 = DIV_ROUND_UP(flits, 2);
1583	/* check how many descriptors needed */
1584	ndesc = DIV_ROUND_UP(flits, 8);
1585
1586	credits = chcr_txq_avail(&q->q) - ndesc;
1587	if (unlikely(credits < 0)) {
1588		chcr_eth_txq_stop(q);
1589		return -ENOMEM;
1590	}
1591
1592	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1593		chcr_eth_txq_stop(q);
1594		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1595	}
1596
1597	last_desc = q->q.pidx + ndesc - 1;
1598	if (last_desc >= q->q.size)
1599		last_desc -= q->q.size;
1600	sgl_sdesc = &q->q.sdesc[last_desc];
1601
1602	if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1603				   sgl_sdesc->addr) < 0)) {
1604		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1605		q->mapping_err++;
1606		return -ENOMEM;
1607	}
1608
1609	iplen = skb_network_header_len(skb);
1610	maclen = skb_mac_header_len(skb);
1611
1612	pos = &q->q.desc[q->q.pidx];
1613	end = (u64 *)pos + flits;
1614	wr = pos;
1615
1616	/* Firmware work request header */
1617	wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1618			       FW_WR_IMMDLEN_V(ctrl));
1619
1620	wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1621	wr->r3 = 0;
1622
1623	cpl = (void *)(wr + 1);
1624
1625	/* CPL header */
1626	cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1627			   TXPKT_INTF_V(tx_info->tx_chan) |
1628			   TXPKT_PF_V(tx_info->adap->pf));
1629	cpl->pack = 0;
1630	cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1631				   TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1632	cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1633		  TXPKT_IPHDR_LEN_V(iplen);
1634	/* checksum offload */
1635	cpl->ctrl1 = cpu_to_be64(cntrl1);
1636	cpl->len = htons(skb->len);
1637
1638	pos = cpl + 1;
1639
1640	cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1641	sgl_sdesc->skb = skb;
1642	chcr_txq_advance(&q->q, ndesc);
1643	skb_tx_timestamp(skb);
1644	cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1645	return 0;
1646}
1647
1648/*
1649 * chcr_ktls_copy_record_in_skb
1650 * @nskb - new skb where the frags to be added.
1651 * @skb - old skb, to copy socket and destructor details.
1652 * @record - specific record which has complete 16k record in frags.
1653 */
1654static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1655					 struct sk_buff *skb,
1656					 struct tls_record_info *record)
1657{
1658	int i = 0;
1659
1660	for (i = 0; i < record->num_frags; i++) {
1661		skb_shinfo(nskb)->frags[i] = record->frags[i];
1662		/* increase the frag ref count */
1663		__skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1664	}
1665
1666	skb_shinfo(nskb)->nr_frags = record->num_frags;
1667	nskb->data_len = record->len;
1668	nskb->len += record->len;
1669	nskb->truesize += record->len;
1670	nskb->sk = skb->sk;
1671	nskb->destructor = skb->destructor;
1672	refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1673}
1674
1675/*
1676 * chcr_end_part_handler: This handler will handle the record which
1677 * is complete or if record's end part is received. T6 adapter has a issue that
1678 * it can't send out TAG with partial record so if its an end part then we have
1679 * to send TAG as well and for which we need to fetch the complete record and
1680 * send it to crypto module.
1681 * @tx_info - driver specific tls info.
1682 * @skb - skb contains partial record.
1683 * @record - complete record of 16K size.
1684 * @tcp_seq
1685 * @mss - segment size in which TP needs to chop a packet.
1686 * @tcp_push_no_fin - tcp push if fin is not set.
1687 * @q - TX queue.
1688 * @tls_end_offset - offset from end of the record.
1689 * @last wr : check if this is the last part of the skb going out.
1690 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1691 */
1692static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1693				 struct sk_buff *skb,
1694				 struct tls_record_info *record,
1695				 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1696				 struct sge_eth_txq *q, u32 skb_offset,
1697				 u32 tls_end_offset, bool last_wr)
1698{
1699	bool free_skb_if_tx_fails = false;
1700	struct sk_buff *nskb = NULL;
1701
1702	/* check if it is a complete record */
1703	if (tls_end_offset == record->len) {
1704		nskb = skb;
1705		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1706	} else {
1707		nskb = alloc_skb(0, GFP_ATOMIC);
1708		if (!nskb) {
1709			dev_kfree_skb_any(skb);
1710			return NETDEV_TX_BUSY;
1711		}
1712
1713		/* copy complete record in skb */
1714		chcr_ktls_copy_record_in_skb(nskb, skb, record);
1715		/* packet is being sent from the beginning, update the tcp_seq
1716		 * accordingly.
1717		 */
1718		tcp_seq = tls_record_start_seq(record);
1719		/* reset skb offset */
1720		skb_offset = 0;
1721
1722		if (last_wr)
1723			dev_kfree_skb_any(skb);
1724		else
1725			free_skb_if_tx_fails = true;
1726
1727		last_wr = true;
1728
1729		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1730	}
1731
1732	if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1733				       last_wr, record->len, skb_offset,
1734				       record->num_frags,
1735				       (last_wr && tcp_push_no_fin),
1736				       mss)) {
1737		if (free_skb_if_tx_fails)
1738			dev_kfree_skb_any(skb);
1739		goto out;
1740	}
1741	tx_info->prev_seq = record->end_seq;
1742	return 0;
1743out:
1744	dev_kfree_skb_any(nskb);
1745	return NETDEV_TX_BUSY;
1746}
1747
1748/*
1749 * chcr_short_record_handler: This handler will take care of the records which
1750 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1751 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1752 * This partial record might be the first part of the record, or the middle
1753 * part. In case of middle record we should fetch the prior data to make it 16
1754 * byte aligned. If it has a partial tls header or iv then get to the start of
1755 * tls header. And if it has partial TAG, then remove the complete TAG and send
1756 * only the payload.
1757 * There is one more possibility that it gets a partial header, send that
1758 * portion as a plaintext.
1759 * @tx_info - driver specific tls info.
1760 * @skb - skb contains partial record..
1761 * @record - complete record of 16K size.
1762 * @tcp_seq
1763 * @mss - segment size in which TP needs to chop a packet.
1764 * @tcp_push_no_fin - tcp push if fin is not set.
1765 * @q - TX queue.
1766 * @tls_end_offset - offset from end of the record.
1767 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1768 */
1769static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1770				     struct sk_buff *skb,
1771				     struct tls_record_info *record,
1772				     u32 tcp_seq, int mss, bool tcp_push_no_fin,
1773				     u32 data_len, u32 skb_offset,
1774				     struct sge_eth_txq *q, u32 tls_end_offset)
1775{
1776	u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1777	u8 prior_data[16] = {0};
1778	u32 prior_data_len = 0;
1779
1780	/* check if the skb is ending in middle of tag/HASH, its a big
1781	 * trouble, send the packet before the HASH.
1782	 */
1783	int remaining_record = tls_end_offset - data_len;
1784
1785	if (remaining_record > 0 &&
1786	    remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1787		int trimmed_len = 0;
1788
1789		if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1790			trimmed_len = data_len -
1791				      (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1792				       remaining_record);
1793		if (!trimmed_len)
1794			return FALLBACK;
1795
1796		WARN_ON(trimmed_len > data_len);
1797
1798		data_len = trimmed_len;
1799		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1800	}
1801
1802	/* check if it is only the header part. */
1803	if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1804		if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1805					  tcp_push_no_fin, q,
1806					  tx_info->port_id, prior_data,
1807					  data_len, skb_offset, prior_data_len))
1808			goto out;
1809
1810		tx_info->prev_seq = tcp_seq + data_len;
1811		return 0;
1812	}
1813
1814	/* check if the middle record's start point is 16 byte aligned. CTR
1815	 * needs 16 byte aligned start point to start encryption.
1816	 */
1817	if (tls_rec_offset) {
1818		/* there is an offset from start, means its a middle record */
1819		int remaining = 0;
1820
1821		if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1822			prior_data_len = tls_rec_offset;
1823			tls_rec_offset = 0;
1824			remaining = 0;
1825		} else {
1826			prior_data_len =
1827				(tls_rec_offset -
1828				(TLS_HEADER_SIZE + tx_info->iv_size))
1829				% AES_BLOCK_LEN;
1830			remaining = tls_rec_offset - prior_data_len;
1831		}
1832
1833		/* if prior_data_len is not zero, means we need to fetch prior
1834		 * data to make this record 16 byte aligned, or we need to reach
1835		 * to start offset.
1836		 */
1837		if (prior_data_len) {
1838			int i = 0;
1839			skb_frag_t *f;
1840			int frag_size = 0, frag_delta = 0;
1841
1842			while (remaining > 0) {
1843				frag_size = skb_frag_size(&record->frags[i]);
1844				if (remaining < frag_size)
1845					break;
1846
1847				remaining -= frag_size;
1848				i++;
1849			}
1850			f = &record->frags[i];
1851			frag_delta = skb_frag_size(f) - remaining;
1852
1853			if (frag_delta >= prior_data_len) {
1854				memcpy_from_page(prior_data, skb_frag_page(f),
1855						 skb_frag_off(f) + remaining,
1856						 prior_data_len);
1857			} else {
1858				memcpy_from_page(prior_data, skb_frag_page(f),
1859						 skb_frag_off(f) + remaining,
1860						 frag_delta);
1861
1862				/* get the next page */
1863				f = &record->frags[i + 1];
1864
1865				memcpy_from_page(prior_data + frag_delta,
1866						 skb_frag_page(f),
1867						 skb_frag_off(f),
1868						 prior_data_len - frag_delta);
1869			}
1870			/* reset tcp_seq as per the prior_data_required len */
1871			tcp_seq -= prior_data_len;
1872		}
1873		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1874	} else {
1875		atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1876	}
1877
1878	if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1879				    mss, tls_rec_offset, prior_data,
1880				    prior_data_len, data_len, skb_offset)) {
1881		goto out;
1882	}
1883
1884	tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1885	return 0;
1886out:
1887	dev_kfree_skb_any(skb);
1888	return NETDEV_TX_BUSY;
1889}
1890
1891static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1892				 struct chcr_ktls_info *tx_info,
1893				 struct sge_eth_txq *q)
1894{
1895	u32 data_len, skb_offset;
1896	struct sk_buff *nskb;
1897	struct tcphdr *th;
1898
1899	nskb = tls_encrypt_skb(skb);
1900
1901	if (!nskb)
1902		return 0;
1903
1904	th = tcp_hdr(nskb);
1905	skb_offset = skb_tcp_all_headers(nskb);
1906	data_len = nskb->len - skb_offset;
1907
1908	if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1909		goto out;
1910
1911	tx_info->prev_seq = ntohl(th->seq) + data_len;
1912	atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1913	return 0;
1914out:
1915	dev_kfree_skb_any(nskb);
1916	return 0;
1917}
1918/* nic tls TX handler */
1919static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1920{
1921	u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1922	struct ch_ktls_port_stats_debug *port_stats;
1923	struct tls_offload_context_tx *tx_ctx;
1924	struct ch_ktls_stats_debug *stats;
1925	struct tcphdr *th = tcp_hdr(skb);
1926	int data_len, qidx, ret = 0, mss;
1927	struct tls_record_info *record;
1928	struct chcr_ktls_info *tx_info;
1929	struct net_device *tls_netdev;
1930	struct tls_context *tls_ctx;
1931	struct sge_eth_txq *q;
1932	struct adapter *adap;
1933	unsigned long flags;
1934
1935	tcp_seq = ntohl(th->seq);
1936	skb_offset = skb_tcp_all_headers(skb);
1937	skb_data_len = skb->len - skb_offset;
1938	data_len = skb_data_len;
1939
1940	mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1941
1942	tls_ctx = tls_get_ctx(skb->sk);
1943	tx_ctx = tls_offload_ctx_tx(tls_ctx);
1944	tls_netdev = rcu_dereference_bh(tls_ctx->netdev);
1945	/* Don't quit on NULL: if tls_device_down is running in parallel,
1946	 * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was
1947	 * true. Rather continue processing this packet.
1948	 */
1949	if (unlikely(tls_netdev && tls_netdev != dev))
1950		goto out;
1951
1952	tx_info = chcr_get_ktls_tx_info(tls_ctx);
1953
1954	if (unlikely(!tx_info))
1955		goto out;
1956
1957	adap = tx_info->adap;
1958	stats = &adap->ch_ktls_stats;
1959	port_stats = &stats->ktls_port[tx_info->port_id];
1960
1961	qidx = skb->queue_mapping;
1962	q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1963	cxgb4_reclaim_completed_tx(adap, &q->q, true);
1964	/* if tcp options are set but finish is not send the options first */
1965	if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1966		ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1967						  tx_info->tx_chan);
1968		if (ret)
1969			return NETDEV_TX_BUSY;
1970	}
1971
1972	/* TCP segments can be in received either complete or partial.
1973	 * chcr_end_part_handler will handle cases if complete record or end
1974	 * part of the record is received. In case of partial end part of record,
1975	 * we will send the complete record again.
1976	 */
1977
1978	spin_lock_irqsave(&tx_ctx->lock, flags);
1979
1980	do {
1981
1982		cxgb4_reclaim_completed_tx(adap, &q->q, true);
1983		/* fetch the tls record */
1984		record = tls_get_record(tx_ctx, tcp_seq,
1985					&tx_info->record_no);
1986		/* By the time packet reached to us, ACK is received, and record
1987		 * won't be found in that case, handle it gracefully.
1988		 */
1989		if (unlikely(!record)) {
1990			spin_unlock_irqrestore(&tx_ctx->lock, flags);
1991			atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
1992			goto out;
1993		}
1994
1995		tls_end_offset = record->end_seq - tcp_seq;
1996
1997		pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1998			 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1999		/* update tcb for the skb */
2000		if (skb_data_len == data_len) {
2001			u32 tx_max = tcp_seq;
2002
2003			if (!tls_record_is_start_marker(record) &&
2004			    tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2005				tx_max = record->end_seq -
2006					TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2007
2008			ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2009						      ntohl(th->ack_seq),
2010						      ntohs(th->window),
2011						      tls_end_offset !=
2012						      record->len);
2013			if (ret) {
2014				spin_unlock_irqrestore(&tx_ctx->lock,
2015						       flags);
2016				goto out;
2017			}
2018
2019			if (th->fin)
2020				skb_get(skb);
2021		}
2022
2023		if (unlikely(tls_record_is_start_marker(record))) {
2024			atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2025			/* If tls_end_offset < data_len, means there is some
2026			 * data after start marker, which needs encryption, send
2027			 * plaintext first and take skb refcount. else send out
2028			 * complete pkt as plaintext.
2029			 */
2030			if (tls_end_offset < data_len)
2031				skb_get(skb);
2032			else
2033				tls_end_offset = data_len;
2034
2035			ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2036						    (!th->fin && th->psh), q,
2037						    tx_info->port_id, NULL,
2038						    tls_end_offset, skb_offset,
2039						    0);
2040
2041			if (ret) {
2042				/* free the refcount taken earlier */
2043				if (tls_end_offset < data_len)
2044					dev_kfree_skb_any(skb);
2045				spin_unlock_irqrestore(&tx_ctx->lock, flags);
2046				goto out;
2047			}
2048
2049			data_len -= tls_end_offset;
2050			tcp_seq = record->end_seq;
2051			skb_offset += tls_end_offset;
2052			continue;
2053		}
2054
2055		/* if a tls record is finishing in this SKB */
2056		if (tls_end_offset <= data_len) {
2057			ret = chcr_end_part_handler(tx_info, skb, record,
2058						    tcp_seq, mss,
2059						    (!th->fin && th->psh), q,
2060						    skb_offset,
2061						    tls_end_offset,
2062						    skb_offset +
2063						    tls_end_offset == skb->len);
2064
2065			data_len -= tls_end_offset;
2066			/* tcp_seq increment is required to handle next record.
2067			 */
2068			tcp_seq += tls_end_offset;
2069			skb_offset += tls_end_offset;
2070		} else {
2071			ret = chcr_short_record_handler(tx_info, skb,
2072							record, tcp_seq, mss,
2073							(!th->fin && th->psh),
2074							data_len, skb_offset,
2075							q, tls_end_offset);
2076			data_len = 0;
2077		}
2078
2079		/* if any failure, come out from the loop. */
2080		if (ret) {
2081			spin_unlock_irqrestore(&tx_ctx->lock, flags);
2082			if (th->fin)
2083				dev_kfree_skb_any(skb);
2084
2085			if (ret == FALLBACK)
2086				return chcr_ktls_sw_fallback(skb, tx_info, q);
2087
2088			return NETDEV_TX_OK;
2089		}
2090
2091		/* length should never be less than 0 */
2092		WARN_ON(data_len < 0);
2093
2094	} while (data_len > 0);
2095
2096	spin_unlock_irqrestore(&tx_ctx->lock, flags);
2097	atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2098	atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2099
2100	/* tcp finish is set, send a separate tcp msg including all the options
2101	 * as well.
2102	 */
2103	if (th->fin) {
2104		chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2105		dev_kfree_skb_any(skb);
2106	}
2107
2108	return NETDEV_TX_OK;
2109out:
2110	dev_kfree_skb_any(skb);
2111	return NETDEV_TX_OK;
2112}
2113
2114static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2115{
2116	struct chcr_ktls_uld_ctx *u_ctx;
2117
2118	pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2119		     CHCR_KTLS_DRV_VERSION);
2120	u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2121	if (!u_ctx) {
2122		u_ctx = ERR_PTR(-ENOMEM);
2123		goto out;
2124	}
2125	u_ctx->lldi = *lldi;
2126	u_ctx->detach = false;
2127	xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH);
2128out:
2129	return u_ctx;
2130}
2131
2132static const struct tlsdev_ops chcr_ktls_ops = {
2133	.tls_dev_add = chcr_ktls_dev_add,
2134	.tls_dev_del = chcr_ktls_dev_del,
2135};
2136
2137static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2138	[CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2139	[CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2140};
2141
2142static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2143				    const struct pkt_gl *pgl)
2144{
2145	const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2146	struct chcr_ktls_uld_ctx *u_ctx = handle;
2147	u8 opcode = rpl->ot.opcode;
2148	struct adapter *adap;
2149
2150	adap = pci_get_drvdata(u_ctx->lldi.pdev);
2151
2152	if (!work_handlers[opcode]) {
2153		pr_err("Unsupported opcode %d received\n", opcode);
2154		return 0;
2155	}
2156
2157	work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2158	return 0;
2159}
2160
2161static void clear_conn_resources(struct chcr_ktls_info *tx_info)
2162{
2163	/* clear l2t entry */
2164	if (tx_info->l2te)
2165		cxgb4_l2t_release(tx_info->l2te);
2166
2167#if IS_ENABLED(CONFIG_IPV6)
2168	/* clear clip entry */
2169	if (tx_info->ip_family == AF_INET6)
2170		cxgb4_clip_release(tx_info->netdev, (const u32 *)
2171				   &tx_info->sk->sk_v6_rcv_saddr,
2172				   1);
2173#endif
2174
2175	/* clear tid */
2176	if (tx_info->tid != -1)
2177		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
2178				 tx_info->tid, tx_info->ip_family);
2179}
2180
2181static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx)
2182{
2183	struct ch_ktls_port_stats_debug *port_stats;
2184	struct tls_offload_context_tx *tx_ctx;
2185	struct chcr_ktls_info *tx_info;
2186	unsigned long index;
2187
2188	xa_for_each(&u_ctx->tid_list, index, tx_ctx) {
2189		tx_info = __chcr_get_ktls_tx_info(tx_ctx);
2190		clear_conn_resources(tx_info);
2191		port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
2192		atomic64_inc(&port_stats->ktls_tx_connection_close);
2193		kvfree(tx_info);
2194		memset(tx_ctx->driver_state, 0, TLS_DRIVER_STATE_SIZE_TX);
2195		/* release module refcount */
2196		module_put(THIS_MODULE);
2197	}
2198}
2199
2200static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2201{
2202	struct chcr_ktls_uld_ctx *u_ctx = handle;
2203
2204	switch (new_state) {
2205	case CXGB4_STATE_UP:
2206		pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2207		mutex_lock(&dev_mutex);
2208		list_add_tail(&u_ctx->entry, &uld_ctx_list);
2209		mutex_unlock(&dev_mutex);
2210		break;
2211	case CXGB4_STATE_START_RECOVERY:
2212	case CXGB4_STATE_DOWN:
2213	case CXGB4_STATE_DETACH:
2214		pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2215		mutex_lock(&dev_mutex);
2216		u_ctx->detach = true;
2217		list_del(&u_ctx->entry);
2218		ch_ktls_reset_all_conn(u_ctx);
2219		xa_destroy(&u_ctx->tid_list);
2220		mutex_unlock(&dev_mutex);
2221		break;
2222	default:
2223		break;
2224	}
2225
2226	return 0;
2227}
2228
2229static struct cxgb4_uld_info chcr_ktls_uld_info = {
2230	.name = CHCR_KTLS_DRV_MODULE_NAME,
2231	.nrxq = 1,
2232	.rxq_size = 1024,
2233	.add = chcr_ktls_uld_add,
2234	.tx_handler = chcr_ktls_xmit,
2235	.rx_handler = chcr_ktls_uld_rx_handler,
2236	.state_change = chcr_ktls_uld_state_change,
2237	.tlsdev_ops = &chcr_ktls_ops,
2238};
2239
2240static int __init chcr_ktls_init(void)
2241{
2242	cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2243	return 0;
2244}
2245
2246static void __exit chcr_ktls_exit(void)
2247{
2248	struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2249	struct adapter *adap;
2250
2251	pr_info("driver unloaded\n");
2252
2253	mutex_lock(&dev_mutex);
2254	list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2255		adap = pci_get_drvdata(u_ctx->lldi.pdev);
2256		memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2257		list_del(&u_ctx->entry);
2258		xa_destroy(&u_ctx->tid_list);
2259		kfree(u_ctx);
2260	}
2261	mutex_unlock(&dev_mutex);
2262	cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2263}
2264
2265module_init(chcr_ktls_init);
2266module_exit(chcr_ktls_exit);
2267
2268MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2269MODULE_LICENSE("GPL");
2270MODULE_AUTHOR("Chelsio Communications");
2271MODULE_VERSION(CHCR_KTLS_DRV_VERSION);