chcr_ipsec.c

  1/*
  2 * This file is part of the Chelsio T6 Crypto driver for Linux.
  3 *
  4 * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
  5 *
  6 * This software is available to you under a choice of one of two
  7 * licenses.  You may choose to be licensed under the terms of the GNU
  8 * General Public License (GPL) Version 2, available from the file
  9 * COPYING in the main directory of this source tree, or the
 10 * OpenIB.org BSD license below:
 11 *
 12 *     Redistribution and use in source and binary forms, with or
 13 *     without modification, are permitted provided that the following
 14 *     conditions are met:
 15 *
 16 *      - Redistributions of source code must retain the above
 17 *        copyright notice, this list of conditions and the following
 18 *        disclaimer.
 19 *
 20 *      - Redistributions in binary form must reproduce the above
 21 *        copyright notice, this list of conditions and the following
 22 *        disclaimer in the documentation and/or other materials
 23 *        provided with the distribution.
 24 *
 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 32 * SOFTWARE.
 33 *
 34 * Written and Maintained by:
 35 *	Atul Gupta (atul.gupta@chelsio.com)
 36 */
 37
 38#define pr_fmt(fmt) "ch_ipsec: " fmt
 39
 40#include <linux/kernel.h>
 41#include <linux/module.h>
 42#include <linux/skbuff.h>
 43#include <linux/rtnetlink.h>
 44#include <linux/highmem.h>
 45#include <linux/if_vlan.h>
 46#include <linux/ip.h>
 47#include <linux/netdevice.h>
 48#include <net/esp.h>
 49#include <net/xfrm.h>
 50#include <crypto/aes.h>
 51#include <crypto/hash.h>
 52#include <crypto/sha1.h>
 53#include <crypto/sha2.h>
 54#include <crypto/authenc.h>
 55#include <crypto/internal/aead.h>
 56#include <crypto/null.h>
 57#include <crypto/internal/skcipher.h>
 58#include <crypto/aead.h>
 59#include <crypto/scatterwalk.h>
 60#include <crypto/internal/hash.h>
 61
 62#include "chcr_ipsec.h"
 63
 64/*
 65 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
 66 * into a WR.
 67 */
 68#define MAX_IMM_TX_PKT_LEN 256
 69#define GCM_ESP_IV_SIZE     8
 70
 71static LIST_HEAD(uld_ctx_list);
 72static DEFINE_MUTEX(dev_mutex);
 73
 74static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state);
 75static int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
 76static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop);
 77static void ch_ipsec_advance_esn_state(struct xfrm_state *x);
 78static void ch_ipsec_xfrm_free_state(struct net_device *dev,
 79				     struct xfrm_state *x);
 80static void ch_ipsec_xfrm_del_state(struct net_device *dev,
 81				    struct xfrm_state *x);
 82static int ch_ipsec_xfrm_add_state(struct net_device *dev,
 83				   struct xfrm_state *x,
 84				   struct netlink_ext_ack *extack);
 85
 86static const struct xfrmdev_ops ch_ipsec_xfrmdev_ops = {
 87	.xdo_dev_state_add      = ch_ipsec_xfrm_add_state,
 88	.xdo_dev_state_delete   = ch_ipsec_xfrm_del_state,
 89	.xdo_dev_state_free     = ch_ipsec_xfrm_free_state,
 90	.xdo_dev_state_advance_esn = ch_ipsec_advance_esn_state,
 91};
 92
 93static struct cxgb4_uld_info ch_ipsec_uld_info = {
 94	.name = CHIPSEC_DRV_MODULE_NAME,
 95	.add = ch_ipsec_uld_add,
 96	.state_change = ch_ipsec_uld_state_change,
 97	.tx_handler = ch_ipsec_xmit,
 98	.xfrmdev_ops = &ch_ipsec_xfrmdev_ops,
 99};
100
101static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop)
102{
103	struct ipsec_uld_ctx *u_ctx;
104
105	pr_info_once("%s - version %s\n", CHIPSEC_DRV_DESC,
106		     CHIPSEC_DRV_VERSION);
107	u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
108	if (!u_ctx) {
109		u_ctx = ERR_PTR(-ENOMEM);
110		goto out;
111	}
112	u_ctx->lldi = *infop;
113out:
114	return u_ctx;
115}
116
117static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state)
118{
119	struct ipsec_uld_ctx *u_ctx = handle;
120
121	pr_debug("new_state %u\n", new_state);
122	switch (new_state) {
123	case CXGB4_STATE_UP:
124		pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
125		mutex_lock(&dev_mutex);
126		list_add_tail(&u_ctx->entry, &uld_ctx_list);
127		mutex_unlock(&dev_mutex);
128		break;
129	case CXGB4_STATE_START_RECOVERY:
130	case CXGB4_STATE_DOWN:
131	case CXGB4_STATE_DETACH:
132		pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
133		list_del(&u_ctx->entry);
134		break;
135	default:
136		break;
137	}
138
139	return 0;
140}
141
142static int ch_ipsec_setauthsize(struct xfrm_state *x,
143				struct ipsec_sa_entry *sa_entry)
144{
145	int hmac_ctrl;
146	int authsize = x->aead->alg_icv_len / 8;
147
148	sa_entry->authsize = authsize;
149
150	switch (authsize) {
151	case ICV_8:
152		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
153		break;
154	case ICV_12:
155		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
156		break;
157	case ICV_16:
158		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
159		break;
160	default:
161		return -EINVAL;
162	}
163	return hmac_ctrl;
164}
165
166static int ch_ipsec_setkey(struct xfrm_state *x,
167			   struct ipsec_sa_entry *sa_entry)
168{
169	int keylen = (x->aead->alg_key_len + 7) / 8;
170	unsigned char *key = x->aead->alg_key;
171	int ck_size, key_ctx_size = 0;
172	unsigned char ghash_h[AEAD_H_SIZE];
173	struct crypto_aes_ctx aes;
174	int ret = 0;
175
176	if (keylen > 3) {
177		keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
178		memcpy(sa_entry->salt, key + keylen, 4);
179	}
180
181	if (keylen == AES_KEYSIZE_128) {
182		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
183	} else if (keylen == AES_KEYSIZE_192) {
184		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
185	} else if (keylen == AES_KEYSIZE_256) {
186		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
187	} else {
188		pr_err("GCM: Invalid key length %d\n", keylen);
189		ret = -EINVAL;
190		goto out;
191	}
192
193	memcpy(sa_entry->key, key, keylen);
194	sa_entry->enckey_len = keylen;
195	key_ctx_size = sizeof(struct _key_ctx) +
196			      ((DIV_ROUND_UP(keylen, 16)) << 4) +
197			      AEAD_H_SIZE;
198
199	sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
200						 CHCR_KEYCTX_MAC_KEY_SIZE_128,
201						 0, 0,
202						 key_ctx_size >> 4);
203
204	/* Calculate the H = CIPH(K, 0 repeated 16 times).
205	 * It will go in key context
206	 */
207	ret = aes_expandkey(&aes, key, keylen);
208	if (ret) {
209		sa_entry->enckey_len = 0;
210		goto out;
211	}
212	memset(ghash_h, 0, AEAD_H_SIZE);
213	aes_encrypt(&aes, ghash_h, ghash_h);
214	memzero_explicit(&aes, sizeof(aes));
215
216	memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
217	       16), ghash_h, AEAD_H_SIZE);
218	sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
219			      AEAD_H_SIZE;
220out:
221	return ret;
222}
223
224/*
225 * ch_ipsec_xfrm_add_state
226 * returns 0 on success, negative error if failed to send message to FPGA
227 * positive error if FPGA returned a bad response
228 */
229static int ch_ipsec_xfrm_add_state(struct net_device *dev,
230				   struct xfrm_state *x,
231				   struct netlink_ext_ack *extack)
232{
233	struct ipsec_sa_entry *sa_entry;
234	int res = 0;
235
236	if (x->props.aalgo != SADB_AALG_NONE) {
237		NL_SET_ERR_MSG_MOD(extack, "Cannot offload authenticated xfrm states");
238		return -EINVAL;
239	}
240	if (x->props.calgo != SADB_X_CALG_NONE) {
241		NL_SET_ERR_MSG_MOD(extack, "Cannot offload compressed xfrm states");
242		return -EINVAL;
243	}
244	if (x->props.family != AF_INET &&
245	    x->props.family != AF_INET6) {
246		NL_SET_ERR_MSG_MOD(extack, "Only IPv4/6 xfrm state offloaded");
247		return -EINVAL;
248	}
249	if (x->props.mode != XFRM_MODE_TRANSPORT &&
250	    x->props.mode != XFRM_MODE_TUNNEL) {
251		NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm offload");
252		return -EINVAL;
253	}
254	if (x->id.proto != IPPROTO_ESP) {
255		NL_SET_ERR_MSG_MOD(extack, "Only ESP xfrm state offloaded");
256		return -EINVAL;
257	}
258	if (x->encap) {
259		NL_SET_ERR_MSG_MOD(extack, "Encapsulated xfrm state not offloaded");
260		return -EINVAL;
261	}
262	if (!x->aead) {
263		NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without aead");
264		return -EINVAL;
265	}
266	if (x->aead->alg_icv_len != 128 &&
267	    x->aead->alg_icv_len != 96) {
268		NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with AEAD ICV length other than 96b & 128b");
269		return -EINVAL;
270	}
271	if ((x->aead->alg_key_len != 128 + 32) &&
272	    (x->aead->alg_key_len != 256 + 32)) {
273		NL_SET_ERR_MSG_MOD(extack, "cannot offload xfrm states with AEAD key length other than 128/256 bit");
274		return -EINVAL;
275	}
276	if (x->tfcpad) {
277		NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with tfc padding");
278		return -EINVAL;
279	}
280	if (!x->geniv) {
281		NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without geniv");
282		return -EINVAL;
283	}
284	if (strcmp(x->geniv, "seqiv")) {
285		NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with geniv other than seqiv");
286		return -EINVAL;
287	}
288	if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
289		NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload");
290		return -EINVAL;
291	}
292
293	if (unlikely(!try_module_get(THIS_MODULE))) {
294		NL_SET_ERR_MSG_MOD(extack, "Failed to acquire module reference");
295		return -ENODEV;
296	}
297
298	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
299	if (!sa_entry) {
300		res = -ENOMEM;
301		module_put(THIS_MODULE);
302		goto out;
303	}
304
305	sa_entry->hmac_ctrl = ch_ipsec_setauthsize(x, sa_entry);
306	if (x->props.flags & XFRM_STATE_ESN)
307		sa_entry->esn = 1;
308	ch_ipsec_setkey(x, sa_entry);
309	x->xso.offload_handle = (unsigned long)sa_entry;
310out:
311	return res;
312}
313
314static void ch_ipsec_xfrm_del_state(struct net_device *dev,
315				    struct xfrm_state *x)
316{
317	/* do nothing */
318	if (!x->xso.offload_handle)
319		return;
320}
321
322static void ch_ipsec_xfrm_free_state(struct net_device *dev,
323				     struct xfrm_state *x)
324{
325	struct ipsec_sa_entry *sa_entry;
326
327	if (!x->xso.offload_handle)
328		return;
329
330	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
331	kfree(sa_entry);
332	module_put(THIS_MODULE);
333}
334
335static void ch_ipsec_advance_esn_state(struct xfrm_state *x)
336{
337	/* do nothing */
338	if (!x->xso.offload_handle)
339		return;
340}
341
342static int is_eth_imm(const struct sk_buff *skb,
343		      struct ipsec_sa_entry *sa_entry)
344{
345	unsigned int kctx_len;
346	int hdrlen;
347
348	kctx_len = sa_entry->kctx_len;
349	hdrlen = sizeof(struct fw_ulptx_wr) +
350		 sizeof(struct chcr_ipsec_req) + kctx_len;
351
352	hdrlen += sizeof(struct cpl_tx_pkt);
353	if (sa_entry->esn)
354		hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16)
355			   << 4);
356	if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
357		return hdrlen;
358	return 0;
359}
360
361static unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
362				      struct ipsec_sa_entry *sa_entry,
363				      bool *immediate)
364{
365	unsigned int kctx_len;
366	unsigned int flits;
367	int aadivlen;
368	int hdrlen;
369
370	kctx_len = sa_entry->kctx_len;
371	hdrlen = is_eth_imm(skb, sa_entry);
372	aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
373						16) : 0;
374	aadivlen <<= 4;
375
376	/* If the skb is small enough, we can pump it out as a work request
377	 * with only immediate data.  In that case we just have to have the
378	 * TX Packet header plus the skb data in the Work Request.
379	 */
380
381	if (hdrlen) {
382		*immediate = true;
383		return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
384	}
385
386	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
387
388	/* Otherwise, we're going to have to construct a Scatter gather list
389	 * of the skb body and fragments.  We also include the flits necessary
390	 * for the TX Packet Work Request and CPL.  We always have a firmware
391	 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
392	 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
393	 * message or, if we're doing a Large Send Offload, an LSO CPL message
394	 * with an embedded TX Packet Write CPL message.
395	 */
396	flits += (sizeof(struct fw_ulptx_wr) +
397		  sizeof(struct chcr_ipsec_req) +
398		  kctx_len +
399		  sizeof(struct cpl_tx_pkt_core) +
400		  aadivlen) / sizeof(__be64);
401	return flits;
402}
403
404static void *copy_esn_pktxt(struct sk_buff *skb,
405			    struct net_device *dev,
406			    void *pos,
407			    struct ipsec_sa_entry *sa_entry)
408{
409	struct chcr_ipsec_aadiv *aadiv;
410	struct ulptx_idata *sc_imm;
411	struct ip_esp_hdr *esphdr;
412	struct xfrm_offload *xo;
413	struct sge_eth_txq *q;
414	struct adapter *adap;
415	struct port_info *pi;
416	__be64 seqno;
417	u32 qidx;
418	u32 seqlo;
419	u8 *iv;
420	int eoq;
421	int len;
422
423	pi = netdev_priv(dev);
424	adap = pi->adapter;
425	qidx = skb->queue_mapping;
426	q = &adap->sge.ethtxq[qidx + pi->first_qset];
427
428	/* end of queue, reset pos to start of queue */
429	eoq = (void *)q->q.stat - pos;
430	if (!eoq)
431		pos = q->q.desc;
432
433	len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4;
434	memset(pos, 0, len);
435	aadiv = (struct chcr_ipsec_aadiv *)pos;
436	esphdr = (struct ip_esp_hdr *)skb_transport_header(skb);
437	iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
438	xo = xfrm_offload(skb);
439
440	aadiv->spi = (esphdr->spi);
441	seqlo = ntohl(esphdr->seq_no);
442	seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32));
443	memcpy(aadiv->seq_no, &seqno, 8);
444	iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
445	memcpy(aadiv->iv, iv, 8);
446
447	if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) {
448		sc_imm = (struct ulptx_idata *)(pos +
449			  (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
450					sizeof(__be64)) << 3));
451		sc_imm->cmd_more = FILL_CMD_MORE(0);
452		sc_imm->len = cpu_to_be32(skb->len);
453	}
454	pos += len;
455	return pos;
456}
457
458static void *copy_cpltx_pktxt(struct sk_buff *skb,
459			      struct net_device *dev,
460			      void *pos,
461			      struct ipsec_sa_entry *sa_entry)
462{
463	struct cpl_tx_pkt_core *cpl;
464	struct sge_eth_txq *q;
465	struct adapter *adap;
466	struct port_info *pi;
467	u32 ctrl0, qidx;
468	u64 cntrl = 0;
469	int left;
470
471	pi = netdev_priv(dev);
472	adap = pi->adapter;
473	qidx = skb->queue_mapping;
474	q = &adap->sge.ethtxq[qidx + pi->first_qset];
475
476	left = (void *)q->q.stat - pos;
477	if (!left)
478		pos = q->q.desc;
479
480	cpl = (struct cpl_tx_pkt_core *)pos;
481
482	cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
483	ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
484			       TXPKT_PF_V(adap->pf);
485	if (skb_vlan_tag_present(skb)) {
486		q->vlan_ins++;
487		cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
488	}
489
490	cpl->ctrl0 = htonl(ctrl0);
491	cpl->pack = htons(0);
492	cpl->len = htons(skb->len);
493	cpl->ctrl1 = cpu_to_be64(cntrl);
494
495	pos += sizeof(struct cpl_tx_pkt_core);
496	/* Copy ESN info for HW */
497	if (sa_entry->esn)
498		pos = copy_esn_pktxt(skb, dev, pos, sa_entry);
499	return pos;
500}
501
502static void *copy_key_cpltx_pktxt(struct sk_buff *skb,
503				  struct net_device *dev,
504				  void *pos,
505				  struct ipsec_sa_entry *sa_entry)
506{
507	struct _key_ctx *key_ctx;
508	int left, eoq, key_len;
509	struct sge_eth_txq *q;
510	struct adapter *adap;
511	struct port_info *pi;
512	unsigned int qidx;
513
514	pi = netdev_priv(dev);
515	adap = pi->adapter;
516	qidx = skb->queue_mapping;
517	q = &adap->sge.ethtxq[qidx + pi->first_qset];
518	key_len = sa_entry->kctx_len;
519
520	/* end of queue, reset pos to start of queue */
521	eoq = (void *)q->q.stat - pos;
522	left = eoq;
523	if (!eoq) {
524		pos = q->q.desc;
525		left = 64 * q->q.size;
526	}
527
528	/* Copy the Key context header */
529	key_ctx = (struct _key_ctx *)pos;
530	key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
531	memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
532	pos += sizeof(struct _key_ctx);
533	left -= sizeof(struct _key_ctx);
534
535	if (likely(key_len <= left)) {
536		memcpy(key_ctx->key, sa_entry->key, key_len);
537		pos += key_len;
538	} else {
539		memcpy(pos, sa_entry->key, left);
540		memcpy(q->q.desc, sa_entry->key + left,
541		       key_len - left);
542		pos = (u8 *)q->q.desc + (key_len - left);
543	}
544	/* Copy CPL TX PKT XT */
545	pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry);
546
547	return pos;
548}
549
550static void *ch_ipsec_crypto_wreq(struct sk_buff *skb,
551				  struct net_device *dev,
552				  void *pos,
553				  int credits,
554				  struct ipsec_sa_entry *sa_entry)
555{
556	struct port_info *pi = netdev_priv(dev);
557	struct adapter *adap = pi->adapter;
558	unsigned int ivsize = GCM_ESP_IV_SIZE;
559	struct chcr_ipsec_wr *wr;
560	bool immediate = false;
561	u16 immdatalen = 0;
562	unsigned int flits;
563	u32 ivinoffset;
564	u32 aadstart;
565	u32 aadstop;
566	u32 ciphstart;
567	u16 sc_more = 0;
568	u32 ivdrop = 0;
569	u32 esnlen = 0;
570	u32 wr_mid;
571	u16 ndesc;
572	int qidx = skb_get_queue_mapping(skb);
573	struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
574	unsigned int kctx_len = sa_entry->kctx_len;
575	int qid = q->q.cntxt_id;
576
577	atomic_inc(&adap->ch_ipsec_stats.ipsec_cnt);
578
579	flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
580	ndesc = DIV_ROUND_UP(flits, 2);
581	if (sa_entry->esn)
582		ivdrop = 1;
583
584	if (immediate)
585		immdatalen = skb->len;
586
587	if (sa_entry->esn) {
588		esnlen = sizeof(struct chcr_ipsec_aadiv);
589		if (!skb_is_nonlinear(skb))
590			sc_more  = 1;
591	}
592
593	/* WR Header */
594	wr = (struct chcr_ipsec_wr *)pos;
595	wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
596	wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc);
597
598	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
599		netif_tx_stop_queue(q->txq);
600		q->q.stops++;
601		if (!q->dbqt)
602			wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
603	}
604	wr_mid |= FW_ULPTX_WR_DATA_F;
605	wr->wreq.flowid_len16 = htonl(wr_mid);
606
607	/* ULPTX */
608	wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
609	wr->req.ulptx.len = htonl(ndesc - 1);
610
611	/* Sub-command */
612	wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more);
613	wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
614					 sizeof(wr->req.key_ctx) +
615					 kctx_len +
616					 sizeof(struct cpl_tx_pkt_core) +
617					 esnlen +
618					 (esnlen ? 0 : immdatalen));
619
620	/* CPL_SEC_PDU */
621	ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) :
622				     (skb_transport_offset(skb) +
623				      sizeof(struct ip_esp_hdr) + 1);
624	wr->req.sec_cpl.op_ivinsrtofst = htonl(
625				CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
626				CPL_TX_SEC_PDU_CPLLEN_V(2) |
627				CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
628				CPL_TX_SEC_PDU_IVINSRTOFST_V(
629							     ivinoffset));
630
631	wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen);
632	aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1);
633	aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET :
634				  (skb_transport_offset(skb) +
635				   sizeof(struct ip_esp_hdr));
636	ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) +
637		    GCM_ESP_IV_SIZE + 1;
638	ciphstart += sa_entry->esn ?  esnlen : 0;
639
640	wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
641							aadstart,
642							aadstop,
643							ciphstart, 0);
644
645	wr->req.sec_cpl.cipherstop_lo_authinsert =
646		FILL_SEC_CPL_AUTHINSERT(0, ciphstart,
647					sa_entry->authsize,
648					 sa_entry->authsize);
649	wr->req.sec_cpl.seqno_numivs =
650		FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
651					 CHCR_SCMD_CIPHER_MODE_AES_GCM,
652					 CHCR_SCMD_AUTH_MODE_GHASH,
653					 sa_entry->hmac_ctrl,
654					 ivsize >> 1);
655	wr->req.sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
656								  0, ivdrop, 0);
657
658	pos += sizeof(struct fw_ulptx_wr) +
659	       sizeof(struct ulp_txpkt) +
660	       sizeof(struct ulptx_idata) +
661	       sizeof(struct cpl_tx_sec_pdu);
662
663	pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
664
665	return pos;
666}
667
668/**
669 *      flits_to_desc - returns the num of Tx descriptors for the given flits
670 *      @n: the number of flits
671 *
672 *      Returns the number of Tx descriptors needed for the supplied number
673 *      of flits.
674 */
675static unsigned int flits_to_desc(unsigned int n)
676{
677	WARN_ON(n > SGE_MAX_WR_LEN / 8);
678	return DIV_ROUND_UP(n, 8);
679}
680
681static unsigned int txq_avail(const struct sge_txq *q)
682{
683	return q->size - 1 - q->in_use;
684}
685
686static void eth_txq_stop(struct sge_eth_txq *q)
687{
688	netif_tx_stop_queue(q->txq);
689	q->q.stops++;
690}
691
692static void txq_advance(struct sge_txq *q, unsigned int n)
693{
694	q->in_use += n;
695	q->pidx += n;
696	if (q->pidx >= q->size)
697		q->pidx -= q->size;
698}
699
700/*
701 *      ch_ipsec_xmit called from ULD Tx handler
702 */
703int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
704{
705	struct xfrm_state *x = xfrm_input_state(skb);
706	unsigned int last_desc, ndesc, flits = 0;
707	struct ipsec_sa_entry *sa_entry;
708	u64 *pos, *end, *before, *sgl;
709	struct tx_sw_desc *sgl_sdesc;
710	int qidx, left, credits;
711	bool immediate = false;
712	struct sge_eth_txq *q;
713	struct adapter *adap;
714	struct port_info *pi;
715	struct sec_path *sp;
716
717	if (!x->xso.offload_handle)
718		return NETDEV_TX_BUSY;
719
720	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
721
722	sp = skb_sec_path(skb);
723	if (sp->len != 1) {
724out_free:       dev_kfree_skb_any(skb);
725		return NETDEV_TX_OK;
726	}
727
728	pi = netdev_priv(dev);
729	adap = pi->adapter;
730	qidx = skb->queue_mapping;
731	q = &adap->sge.ethtxq[qidx + pi->first_qset];
732
733	cxgb4_reclaim_completed_tx(adap, &q->q, true);
734
735	flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
736	ndesc = flits_to_desc(flits);
737	credits = txq_avail(&q->q) - ndesc;
738
739	if (unlikely(credits < 0)) {
740		eth_txq_stop(q);
741		dev_err(adap->pdev_dev,
742			"%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
743			dev->name, qidx, credits, ndesc, txq_avail(&q->q),
744			flits);
745		return NETDEV_TX_BUSY;
746	}
747
748	last_desc = q->q.pidx + ndesc - 1;
749	if (last_desc >= q->q.size)
750		last_desc -= q->q.size;
751	sgl_sdesc = &q->q.sdesc[last_desc];
752
753	if (!immediate &&
754	    unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
755		memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
756		q->mapping_err++;
757		goto out_free;
758	}
759
760	pos = (u64 *)&q->q.desc[q->q.pidx];
761	before = (u64 *)pos;
762	end = (u64 *)pos + flits;
763	/* Setup IPSec CPL */
764	pos = (void *)ch_ipsec_crypto_wreq(skb, dev, (void *)pos,
765					   credits, sa_entry);
766	if (before > (u64 *)pos) {
767		left = (u8 *)end - (u8 *)q->q.stat;
768		end = (void *)q->q.desc + left;
769	}
770	if (pos == (u64 *)q->q.stat) {
771		left = (u8 *)end - (u8 *)q->q.stat;
772		end = (void *)q->q.desc + left;
773		pos = (void *)q->q.desc;
774	}
775
776	sgl = (void *)pos;
777	if (immediate) {
778		cxgb4_inline_tx_skb(skb, &q->q, sgl);
779		dev_consume_skb_any(skb);
780	} else {
781		cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
782				0, sgl_sdesc->addr);
783		skb_orphan(skb);
784		sgl_sdesc->skb = skb;
785	}
786	txq_advance(&q->q, ndesc);
787
788	cxgb4_ring_tx_db(adap, &q->q, ndesc);
789	return NETDEV_TX_OK;
790}
791
792static int __init ch_ipsec_init(void)
793{
794	cxgb4_register_uld(CXGB4_ULD_IPSEC, &ch_ipsec_uld_info);
795
796	return 0;
797}
798
799static void __exit ch_ipsec_exit(void)
800{
801	struct ipsec_uld_ctx *u_ctx, *tmp;
802	struct adapter *adap;
803
804	mutex_lock(&dev_mutex);
805	list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
806		adap = pci_get_drvdata(u_ctx->lldi.pdev);
807		atomic_set(&adap->ch_ipsec_stats.ipsec_cnt, 0);
808		list_del(&u_ctx->entry);
809		kfree(u_ctx);
810	}
811	mutex_unlock(&dev_mutex);
812	cxgb4_unregister_uld(CXGB4_ULD_IPSEC);
813}
814
815module_init(ch_ipsec_init);
816module_exit(ch_ipsec_exit);
817
818MODULE_DESCRIPTION("Crypto IPSEC for Chelsio Terminator cards.");
819MODULE_LICENSE("GPL");
820MODULE_AUTHOR("Chelsio Communications");
821MODULE_VERSION(CHIPSEC_DRV_VERSION);
822