ipsec.c

  1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
  2/* Copyright (C) 2018 Netronome Systems, Inc */
  3/* Copyright (C) 2021 Corigine, Inc */
  4
  5#include <linux/module.h>
  6#include <linux/kernel.h>
  7#include <linux/init.h>
  8#include <linux/netdevice.h>
  9#include <linux/unaligned.h>
 10#include <linux/ktime.h>
 11#include <net/xfrm.h>
 12
 13#include "../nfpcore/nfp_dev.h"
 14#include "../nfp_net_ctrl.h"
 15#include "../nfp_net.h"
 16#include "crypto.h"
 17
 18#define NFP_NET_IPSEC_MAX_SA_CNT  (16 * 1024) /* Firmware support a maximum of 16K SA offload */
 19
 20/* IPsec config message cmd codes */
 21enum nfp_ipsec_cfg_mssg_cmd_codes {
 22	NFP_IPSEC_CFG_MSSG_ADD_SA,	 /* Add a new SA */
 23	NFP_IPSEC_CFG_MSSG_INV_SA	 /* Invalidate an existing SA */
 24};
 25
 26/* IPsec config message response codes */
 27enum nfp_ipsec_cfg_mssg_rsp_codes {
 28	NFP_IPSEC_CFG_MSSG_OK,
 29	NFP_IPSEC_CFG_MSSG_FAILED,
 30	NFP_IPSEC_CFG_MSSG_SA_VALID,
 31	NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED,
 32	NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED,
 33	NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD
 34};
 35
 36/* Protocol */
 37enum nfp_ipsec_sa_prot {
 38	NFP_IPSEC_PROTOCOL_AH = 0,
 39	NFP_IPSEC_PROTOCOL_ESP = 1
 40};
 41
 42/* Mode */
 43enum nfp_ipsec_sa_mode {
 44	NFP_IPSEC_PROTMODE_TRANSPORT = 0,
 45	NFP_IPSEC_PROTMODE_TUNNEL = 1
 46};
 47
 48/* Cipher types */
 49enum nfp_ipsec_sa_cipher {
 50	NFP_IPSEC_CIPHER_NULL,
 51	NFP_IPSEC_CIPHER_3DES,
 52	NFP_IPSEC_CIPHER_AES128,
 53	NFP_IPSEC_CIPHER_AES192,
 54	NFP_IPSEC_CIPHER_AES256,
 55	NFP_IPSEC_CIPHER_AES128_NULL,
 56	NFP_IPSEC_CIPHER_AES192_NULL,
 57	NFP_IPSEC_CIPHER_AES256_NULL,
 58	NFP_IPSEC_CIPHER_CHACHA20
 59};
 60
 61/* Cipher modes */
 62enum nfp_ipsec_sa_cipher_mode {
 63	NFP_IPSEC_CIMODE_ECB,
 64	NFP_IPSEC_CIMODE_CBC,
 65	NFP_IPSEC_CIMODE_CFB,
 66	NFP_IPSEC_CIMODE_OFB,
 67	NFP_IPSEC_CIMODE_CTR
 68};
 69
 70/* Hash types */
 71enum nfp_ipsec_sa_hash_type {
 72	NFP_IPSEC_HASH_NONE,
 73	NFP_IPSEC_HASH_MD5_96,
 74	NFP_IPSEC_HASH_SHA1_96,
 75	NFP_IPSEC_HASH_SHA256_96,
 76	NFP_IPSEC_HASH_SHA384_96,
 77	NFP_IPSEC_HASH_SHA512_96,
 78	NFP_IPSEC_HASH_MD5_128,
 79	NFP_IPSEC_HASH_SHA1_80,
 80	NFP_IPSEC_HASH_SHA256_128,
 81	NFP_IPSEC_HASH_SHA384_192,
 82	NFP_IPSEC_HASH_SHA512_256,
 83	NFP_IPSEC_HASH_GF128_128,
 84	NFP_IPSEC_HASH_POLY1305_128
 85};
 86
 87/* IPSEC_CFG_MSSG_ADD_SA */
 88struct nfp_ipsec_cfg_add_sa {
 89	u32 ciph_key[8];		  /* Cipher Key */
 90	union {
 91		u32 auth_key[16];	  /* Authentication Key */
 92		struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */
 93			u32 salt;	  /* Initialized with SA */
 94			u32 resv[15];
 95		} aesgcm_fields;
 96	};
 97	struct sa_ctrl_word {
 98		uint32_t hash   :4;	  /* From nfp_ipsec_sa_hash_type */
 99		uint32_t cimode :4;	  /* From nfp_ipsec_sa_cipher_mode */
100		uint32_t cipher :4;	  /* From nfp_ipsec_sa_cipher */
101		uint32_t mode   :2;	  /* From nfp_ipsec_sa_mode */
102		uint32_t proto  :2;	  /* From nfp_ipsec_sa_prot */
103		uint32_t dir :1;	  /* SA direction */
104		uint32_t resv0 :12;
105		uint32_t encap_dsbl:1;	  /* Encap/Decap disable */
106		uint32_t resv1 :2;	  /* Must be set to 0 */
107	} ctrl_word;
108	u32 spi;			  /* SPI Value */
109	uint32_t pmtu_limit :16;          /* PMTU Limit */
110	uint32_t resv0 :5;
111	uint32_t ipv6       :1;		  /* Outbound IPv6 addr format */
112	uint32_t resv1	 :10;
113	u32 resv2[2];
114	u32 src_ip[4];			  /* Src IP addr */
115	u32 dst_ip[4];			  /* Dst IP addr */
116	u32 resv3[6];
117};
118
119/* IPSEC_CFG_MSSG */
120struct nfp_ipsec_cfg_mssg {
121	union {
122		struct{
123			uint32_t cmd:16;     /* One of nfp_ipsec_cfg_mssg_cmd_codes */
124			uint32_t rsp:16;     /* One of nfp_ipsec_cfg_mssg_rsp_codes */
125			uint32_t sa_idx:16;  /* SA table index */
126			uint32_t spare0:16;
127			struct nfp_ipsec_cfg_add_sa cfg_add_sa;
128		};
129		u32 raw[64];
130	};
131};
132
133static int nfp_net_ipsec_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry)
134{
135	unsigned int offset = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;
136	struct nfp_ipsec_cfg_mssg *msg = (struct nfp_ipsec_cfg_mssg *)entry->msg;
137	int i, msg_size, ret;
138
139	ret = nfp_net_mbox_lock(nn, sizeof(*msg));
140	if (ret)
141		return ret;
142
143	msg_size = ARRAY_SIZE(msg->raw);
144	for (i = 0; i < msg_size; i++)
145		nn_writel(nn, offset + 4 * i, msg->raw[i]);
146
147	ret = nfp_net_mbox_reconfig(nn, entry->cmd);
148	if (ret < 0) {
149		nn_ctrl_bar_unlock(nn);
150		return ret;
151	}
152
153	/* For now we always read the whole message response back */
154	for (i = 0; i < msg_size; i++)
155		msg->raw[i] = nn_readl(nn, offset + 4 * i);
156
157	nn_ctrl_bar_unlock(nn);
158
159	switch (msg->rsp) {
160	case NFP_IPSEC_CFG_MSSG_OK:
161		return 0;
162	case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD:
163		return -EINVAL;
164	case NFP_IPSEC_CFG_MSSG_SA_VALID:
165		return -EEXIST;
166	case NFP_IPSEC_CFG_MSSG_FAILED:
167	case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED:
168	case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED:
169		return -EIO;
170	default:
171		return -EINVAL;
172	}
173}
174
175static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen)
176{
177	bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC);
178
179	switch (keylen) {
180	case 128:
181		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL :
182						   NFP_IPSEC_CIPHER_AES128;
183		break;
184	case 192:
185		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL :
186						   NFP_IPSEC_CIPHER_AES192;
187		break;
188	case 256:
189		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL :
190						   NFP_IPSEC_CIPHER_AES256;
191		break;
192	default:
193		return -EINVAL;
194	}
195
196	return 0;
197}
198
199static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
200{
201	switch (*trunc_len) {
202	case 96:
203		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96;
204		break;
205	case 128:
206		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128;
207		break;
208	default:
209		*trunc_len = 0;
210	}
211}
212
213static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
214{
215	switch (*trunc_len) {
216	case 96:
217		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96;
218		break;
219	case 80:
220		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80;
221		break;
222	default:
223		*trunc_len = 0;
224	}
225}
226
227static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
228{
229	switch (*trunc_len) {
230	case 96:
231		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96;
232		break;
233	case 128:
234		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128;
235		break;
236	default:
237		*trunc_len = 0;
238	}
239}
240
241static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
242{
243	switch (*trunc_len) {
244	case 96:
245		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96;
246		break;
247	case 192:
248		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192;
249		break;
250	default:
251		*trunc_len = 0;
252	}
253}
254
255static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)
256{
257	switch (*trunc_len) {
258	case 96:
259		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96;
260		break;
261	case 256:
262		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256;
263		break;
264	default:
265		*trunc_len = 0;
266	}
267}
268
269static int nfp_net_xfrm_add_state(struct net_device *dev,
270				  struct xfrm_state *x,
271				  struct netlink_ext_ack *extack)
272{
273	struct nfp_ipsec_cfg_mssg msg = {};
274	int i, key_len, trunc_len, err = 0;
275	struct nfp_ipsec_cfg_add_sa *cfg;
276	struct nfp_net *nn;
277	unsigned int saidx;
278
279	nn = netdev_priv(dev);
280	cfg = &msg.cfg_add_sa;
281
282	/* General */
283	switch (x->props.mode) {
284	case XFRM_MODE_TUNNEL:
285		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL;
286		break;
287	case XFRM_MODE_TRANSPORT:
288		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT;
289		break;
290	default:
291		NL_SET_ERR_MSG_MOD(extack, "Unsupported mode for xfrm offload");
292		return -EINVAL;
293	}
294
295	switch (x->id.proto) {
296	case IPPROTO_ESP:
297		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP;
298		break;
299	case IPPROTO_AH:
300		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH;
301		break;
302	default:
303		NL_SET_ERR_MSG_MOD(extack, "Unsupported protocol for xfrm offload");
304		return -EINVAL;
305	}
306
307	if (x->props.flags & XFRM_STATE_ESN) {
308		NL_SET_ERR_MSG_MOD(extack, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload");
309		return -EINVAL;
310	}
311
312	if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
313		NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload type");
314		return -EINVAL;
315	}
316
317	cfg->spi = ntohl(x->id.spi);
318
319	/* Hash/Authentication */
320	if (x->aalg)
321		trunc_len = x->aalg->alg_trunc_len;
322	else
323		trunc_len = 0;
324
325	switch (x->props.aalgo) {
326	case SADB_AALG_NONE:
327		if (x->aead) {
328			trunc_len = -1;
329		} else {
330			NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
331			return -EINVAL;
332		}
333		break;
334	case SADB_X_AALG_NULL:
335		cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE;
336		trunc_len = -1;
337		break;
338	case SADB_AALG_MD5HMAC:
339		if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) {
340			NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
341			return -EINVAL;
342		}
343		set_md5hmac(cfg, &trunc_len);
344		break;
345	case SADB_AALG_SHA1HMAC:
346		set_sha1hmac(cfg, &trunc_len);
347		break;
348	case SADB_X_AALG_SHA2_256HMAC:
349		set_sha2_256hmac(cfg, &trunc_len);
350		break;
351	case SADB_X_AALG_SHA2_384HMAC:
352		set_sha2_384hmac(cfg, &trunc_len);
353		break;
354	case SADB_X_AALG_SHA2_512HMAC:
355		set_sha2_512hmac(cfg, &trunc_len);
356		break;
357	default:
358		NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm");
359		return -EINVAL;
360	}
361
362	if (!trunc_len) {
363		NL_SET_ERR_MSG_MOD(extack, "Unsupported authentication algorithm trunc length");
364		return -EINVAL;
365	}
366
367	if (x->aalg) {
368		key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE);
369		if (key_len > sizeof(cfg->auth_key)) {
370			NL_SET_ERR_MSG_MOD(extack, "Insufficient space for offloaded auth key");
371			return -EINVAL;
372		}
373		for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++)
374			cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key +
375							      sizeof(cfg->auth_key[0]) * i);
376	}
377
378	/* Encryption */
379	switch (x->props.ealgo) {
380	case SADB_EALG_NONE:
381		/* The xfrm descriptor for CHACAH20_POLY1305 does not set the algorithm id, which
382		 * is the default value SADB_EALG_NONE. In the branch of SADB_EALG_NONE, driver
383		 * uses algorithm name to identify CHACAH20_POLY1305's algorithm.
384		 */
385		if (x->aead && !strcmp(x->aead->alg_name, "rfc7539esp(chacha20,poly1305)")) {
386			if (nn->pdev->device != PCI_DEVICE_ID_NFP3800) {
387				NL_SET_ERR_MSG_MOD(extack,
388						   "Unsupported encryption algorithm for offload");
389				return -EINVAL;
390			}
391			if (x->aead->alg_icv_len != 128) {
392				NL_SET_ERR_MSG_MOD(extack,
393						   "ICV must be 128bit with CHACHA20_POLY1305");
394				return -EINVAL;
395			}
396
397			/* Aead->alg_key_len includes 32-bit salt */
398			if (x->aead->alg_key_len - 32 != 256) {
399				NL_SET_ERR_MSG_MOD(extack, "Unsupported CHACHA20 key length");
400				return -EINVAL;
401			}
402
403			/* The CHACHA20's mode is not configured */
404			cfg->ctrl_word.hash = NFP_IPSEC_HASH_POLY1305_128;
405			cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_CHACHA20;
406			break;
407		}
408		fallthrough;
409	case SADB_EALG_NULL:
410		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
411		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL;
412		break;
413	case SADB_EALG_3DESCBC:
414		if (nn->pdev->device == PCI_DEVICE_ID_NFP3800) {
415			NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload");
416			return -EINVAL;
417		}
418		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
419		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES;
420		break;
421	case SADB_X_EALG_AES_GCM_ICV16:
422	case SADB_X_EALG_NULL_AES_GMAC:
423		if (!x->aead) {
424			NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data");
425			return -EINVAL;
426		}
427
428		if (x->aead->alg_icv_len != 128) {
429			NL_SET_ERR_MSG_MOD(extack, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16");
430			return -EINVAL;
431		}
432		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR;
433		cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128;
434
435		/* Aead->alg_key_len includes 32-bit salt */
436		if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) {
437			NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length");
438			return -EINVAL;
439		}
440		break;
441	case SADB_X_EALG_AESCBC:
442		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;
443		if (!x->ealg) {
444			NL_SET_ERR_MSG_MOD(extack, "Invalid AES key data");
445			return -EINVAL;
446		}
447		if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) {
448			NL_SET_ERR_MSG_MOD(extack, "Unsupported AES key length");
449			return -EINVAL;
450		}
451		break;
452	default:
453		NL_SET_ERR_MSG_MOD(extack, "Unsupported encryption algorithm for offload");
454		return -EINVAL;
455	}
456
457	if (x->aead) {
458		int key_offset = 0;
459		int salt_len = 4;
460
461		key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE);
462		key_len -= salt_len;
463
464		if (key_len > sizeof(cfg->ciph_key)) {
465			NL_SET_ERR_MSG_MOD(extack, "aead: Insufficient space for offloaded key");
466			return -EINVAL;
467		}
468
469		/* The CHACHA20's key order needs to be adjusted based on hardware design.
470		 * Other's key order: {K0, K1, K2, K3, K4, K5, K6, K7}
471		 * CHACHA20's key order: {K4, K5, K6, K7, K0, K1, K2, K3}
472		 */
473		if (!strcmp(x->aead->alg_name, "rfc7539esp(chacha20,poly1305)"))
474			key_offset = key_len / sizeof(cfg->ciph_key[0]) >> 1;
475
476		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]); i++) {
477			int index = (i + key_offset) % (key_len / sizeof(cfg->ciph_key[0]));
478
479			cfg->ciph_key[index] = get_unaligned_be32(x->aead->alg_key +
480								  sizeof(cfg->ciph_key[0]) * i);
481		}
482
483		/* Load up the salt */
484		cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len);
485	}
486
487	if (x->ealg) {
488		key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE);
489
490		if (key_len > sizeof(cfg->ciph_key)) {
491			NL_SET_ERR_MSG_MOD(extack, "ealg: Insufficient space for offloaded key");
492			return -EINVAL;
493		}
494		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)
495			cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key +
496							      sizeof(cfg->ciph_key[0]) * i);
497	}
498
499	/* IP related info */
500	switch (x->props.family) {
501	case AF_INET:
502		cfg->ipv6 = 0;
503		cfg->src_ip[0] = ntohl(x->props.saddr.a4);
504		cfg->dst_ip[0] = ntohl(x->id.daddr.a4);
505		break;
506	case AF_INET6:
507		cfg->ipv6 = 1;
508		for (i = 0; i < 4; i++) {
509			cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]);
510			cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]);
511		}
512		break;
513	default:
514		NL_SET_ERR_MSG_MOD(extack, "Unsupported address family");
515		return -EINVAL;
516	}
517
518	/* Maximum nic IPsec code could handle. Other limits may apply. */
519	cfg->pmtu_limit = 0xffff;
520	cfg->ctrl_word.encap_dsbl = 1;
521
522	/* SA direction */
523	cfg->ctrl_word.dir = x->xso.dir;
524
525	/* Find unused SA data*/
526	err = xa_alloc(&nn->xa_ipsec, &saidx, x,
527		       XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL);
528	if (err < 0) {
529		NL_SET_ERR_MSG_MOD(extack, "Unable to get sa_data number for IPsec");
530		return err;
531	}
532
533	/* Allocate saidx and commit the SA */
534	msg.cmd = NFP_IPSEC_CFG_MSSG_ADD_SA;
535	msg.sa_idx = saidx;
536	err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, &msg,
537					   sizeof(msg), nfp_net_ipsec_cfg);
538	if (err) {
539		xa_erase(&nn->xa_ipsec, saidx);
540		NL_SET_ERR_MSG_MOD(extack, "Failed to issue IPsec command");
541		return err;
542	}
543
544	/* 0 is invalid offload_handle for kernel */
545	x->xso.offload_handle = saidx + 1;
546	return 0;
547}
548
549static void nfp_net_xfrm_del_state(struct net_device *dev, struct xfrm_state *x)
550{
551	struct nfp_ipsec_cfg_mssg msg = {
552		.cmd = NFP_IPSEC_CFG_MSSG_INV_SA,
553		.sa_idx = x->xso.offload_handle - 1,
554	};
555	struct nfp_net *nn;
556	int err;
557
558	nn = netdev_priv(dev);
559	err = nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_IPSEC, &msg,
560					   sizeof(msg), nfp_net_ipsec_cfg);
561	if (err)
562		nn_warn(nn, "Failed to invalidate SA in hardware\n");
563
564	xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1);
565}
566
567static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = {
568	.xdo_dev_state_add = nfp_net_xfrm_add_state,
569	.xdo_dev_state_delete = nfp_net_xfrm_del_state,
570};
571
572void nfp_net_ipsec_init(struct nfp_net *nn)
573{
574	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
575		return;
576
577	xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC);
578	nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops;
579}
580
581void nfp_net_ipsec_clean(struct nfp_net *nn)
582{
583	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))
584		return;
585
586	WARN_ON(!xa_empty(&nn->xa_ipsec));
587	xa_destroy(&nn->xa_ipsec);
588}
589
590bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,
591			   struct nfp_ipsec_offload *offload_info)
592{
593	struct xfrm_offload *xo = xfrm_offload(skb);
594	struct xfrm_state *x;
595
596	x = xfrm_input_state(skb);
597	if (!x)
598		return false;
599
600	offload_info->seq_hi = xo->seq.hi;
601	offload_info->seq_low = xo->seq.low;
602	offload_info->handle = x->xso.offload_handle;
603
604	return true;
605}
606
607int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)
608{
609	struct net_device *netdev = skb->dev;
610	struct xfrm_offload *xo;
611	struct xfrm_state *x;
612	struct sec_path *sp;
613	struct nfp_net *nn;
614	u32 saidx;
615
616	nn = netdev_priv(netdev);
617
618	saidx = meta->ipsec_saidx - 1;
619	if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT)
620		return -EINVAL;
621
622	sp = secpath_set(skb);
623	if (unlikely(!sp))
624		return -ENOMEM;
625
626	xa_lock(&nn->xa_ipsec);
627	x = xa_load(&nn->xa_ipsec, saidx);
628	xa_unlock(&nn->xa_ipsec);
629	if (!x)
630		return -EINVAL;
631
632	xfrm_state_hold(x);
633	sp->xvec[sp->len++] = x;
634	sp->olen++;
635	xo = xfrm_offload(skb);
636	xo->flags = CRYPTO_DONE;
637	xo->status = CRYPTO_SUCCESS;
638
639	return 0;
640}