macsec.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * drivers/net/macsec.c - MACsec device
   4 *
   5 * Copyright (c) 2015 Sabrina Dubroca <sd@queasysnail.net>
   6 */
   7
   8#include <linux/types.h>
   9#include <linux/skbuff.h>
  10#include <linux/socket.h>
  11#include <linux/module.h>
  12#include <crypto/aead.h>
  13#include <linux/etherdevice.h>
  14#include <linux/netdevice.h>
  15#include <linux/rtnetlink.h>
  16#include <linux/refcount.h>
  17#include <net/genetlink.h>
  18#include <net/sock.h>
  19#include <net/gro_cells.h>
  20#include <net/macsec.h>
  21#include <net/dst_metadata.h>
  22#include <net/netdev_lock.h>
  23#include <linux/phy.h>
  24#include <linux/byteorder/generic.h>
  25#include <linux/if_arp.h>
  26
  27#include <uapi/linux/if_macsec.h>
  28
  29/* SecTAG length = macsec_eth_header without the optional SCI */
  30#define MACSEC_TAG_LEN 6
  31
  32struct macsec_eth_header {
  33	struct ethhdr eth;
  34	/* SecTAG */
  35	u8  tci_an;
  36#if defined(__LITTLE_ENDIAN_BITFIELD)
  37	u8  short_length:6,
  38		  unused:2;
  39#elif defined(__BIG_ENDIAN_BITFIELD)
  40	u8        unused:2,
  41	    short_length:6;
  42#else
  43#error	"Please fix <asm/byteorder.h>"
  44#endif
  45	__be32 packet_number;
  46	u8 secure_channel_id[8]; /* optional */
  47} __packed;
  48
  49/* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */
  50#define MIN_NON_SHORT_LEN 48
  51
  52#define GCM_AES_IV_LEN 12
  53
  54#define for_each_rxsc(secy, sc)				\
  55	for (sc = rcu_dereference_bh(secy->rx_sc);	\
  56	     sc;					\
  57	     sc = rcu_dereference_bh(sc->next))
  58#define for_each_rxsc_rtnl(secy, sc)			\
  59	for (sc = rtnl_dereference(secy->rx_sc);	\
  60	     sc;					\
  61	     sc = rtnl_dereference(sc->next))
  62
  63#define pn_same_half(pn1, pn2) (!(((pn1) >> 31) ^ ((pn2) >> 31)))
  64
  65struct gcm_iv_xpn {
  66	union {
  67		u8 short_secure_channel_id[4];
  68		ssci_t ssci;
  69	};
  70	__be64 pn;
  71} __packed;
  72
  73struct gcm_iv {
  74	union {
  75		u8 secure_channel_id[8];
  76		sci_t sci;
  77	};
  78	__be32 pn;
  79};
  80
  81#define MACSEC_VALIDATE_DEFAULT MACSEC_VALIDATE_STRICT
  82
  83struct pcpu_secy_stats {
  84	struct macsec_dev_stats stats;
  85	struct u64_stats_sync syncp;
  86};
  87
  88/**
  89 * struct macsec_dev - private data
  90 * @secy: SecY config
  91 * @real_dev: pointer to underlying netdevice
  92 * @dev_tracker: refcount tracker for @real_dev reference
  93 * @stats: MACsec device stats
  94 * @secys: linked list of SecY's on the underlying device
  95 * @gro_cells: pointer to the Generic Receive Offload cell
  96 * @offload: status of offloading on the MACsec device
  97 * @insert_tx_tag: when offloading, device requires to insert an
  98 *	additional tag
  99 */
 100struct macsec_dev {
 101	struct macsec_secy secy;
 102	struct net_device *real_dev;
 103	netdevice_tracker dev_tracker;
 104	struct pcpu_secy_stats __percpu *stats;
 105	struct list_head secys;
 106	struct gro_cells gro_cells;
 107	enum macsec_offload offload;
 108	bool insert_tx_tag;
 109};
 110
 111/**
 112 * struct macsec_rxh_data - rx_handler private argument
 113 * @secys: linked list of SecY's on this underlying device
 114 */
 115struct macsec_rxh_data {
 116	struct list_head secys;
 117};
 118
 119static struct macsec_dev *macsec_priv(const struct net_device *dev)
 120{
 121	return (struct macsec_dev *)netdev_priv(dev);
 122}
 123
 124static struct macsec_rxh_data *macsec_data_rcu(const struct net_device *dev)
 125{
 126	return rcu_dereference_bh(dev->rx_handler_data);
 127}
 128
 129static struct macsec_rxh_data *macsec_data_rtnl(const struct net_device *dev)
 130{
 131	return rtnl_dereference(dev->rx_handler_data);
 132}
 133
 134struct macsec_cb {
 135	struct aead_request *req;
 136	union {
 137		struct macsec_tx_sa *tx_sa;
 138		struct macsec_rx_sa *rx_sa;
 139	};
 140	u8 assoc_num;
 141	bool valid;
 142	bool has_sci;
 143};
 144
 145static struct macsec_rx_sa *macsec_rxsa_get(struct macsec_rx_sa __rcu *ptr)
 146{
 147	struct macsec_rx_sa *sa = rcu_dereference_bh(ptr);
 148
 149	if (!sa || !sa->active)
 150		return NULL;
 151
 152	if (!refcount_inc_not_zero(&sa->refcnt))
 153		return NULL;
 154
 155	return sa;
 156}
 157
 158static void free_rx_sc_rcu(struct rcu_head *head)
 159{
 160	struct macsec_rx_sc *rx_sc = container_of(head, struct macsec_rx_sc, rcu_head);
 161
 162	free_percpu(rx_sc->stats);
 163	kfree(rx_sc);
 164}
 165
 166static struct macsec_rx_sc *macsec_rxsc_get(struct macsec_rx_sc *sc)
 167{
 168	return refcount_inc_not_zero(&sc->refcnt) ? sc : NULL;
 169}
 170
 171static void macsec_rxsc_put(struct macsec_rx_sc *sc)
 172{
 173	if (refcount_dec_and_test(&sc->refcnt))
 174		call_rcu(&sc->rcu_head, free_rx_sc_rcu);
 175}
 176
 177static void free_rxsa(struct rcu_head *head)
 178{
 179	struct macsec_rx_sa *sa = container_of(head, struct macsec_rx_sa, rcu);
 180
 181	crypto_free_aead(sa->key.tfm);
 182	free_percpu(sa->stats);
 183	kfree(sa);
 184}
 185
 186static void macsec_rxsa_put(struct macsec_rx_sa *sa)
 187{
 188	if (refcount_dec_and_test(&sa->refcnt))
 189		call_rcu(&sa->rcu, free_rxsa);
 190}
 191
 192static struct macsec_tx_sa *macsec_txsa_get(struct macsec_tx_sa __rcu *ptr)
 193{
 194	struct macsec_tx_sa *sa = rcu_dereference_bh(ptr);
 195
 196	if (!sa || !sa->active)
 197		return NULL;
 198
 199	if (!refcount_inc_not_zero(&sa->refcnt))
 200		return NULL;
 201
 202	return sa;
 203}
 204
 205static void free_txsa(struct rcu_head *head)
 206{
 207	struct macsec_tx_sa *sa = container_of(head, struct macsec_tx_sa, rcu);
 208
 209	crypto_free_aead(sa->key.tfm);
 210	free_percpu(sa->stats);
 211	kfree(sa);
 212}
 213
 214static void macsec_txsa_put(struct macsec_tx_sa *sa)
 215{
 216	if (refcount_dec_and_test(&sa->refcnt))
 217		call_rcu(&sa->rcu, free_txsa);
 218}
 219
 220static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
 221{
 222	BUILD_BUG_ON(sizeof(struct macsec_cb) > sizeof(skb->cb));
 223	return (struct macsec_cb *)skb->cb;
 224}
 225
 226#define MACSEC_PORT_SCB (0x0000)
 227#define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
 228#define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff)
 229
 230#define MACSEC_GCM_AES_128_SAK_LEN 16
 231#define MACSEC_GCM_AES_256_SAK_LEN 32
 232
 233#define DEFAULT_SAK_LEN MACSEC_GCM_AES_128_SAK_LEN
 234#define DEFAULT_XPN false
 235#define DEFAULT_SEND_SCI true
 236#define DEFAULT_ENCRYPT false
 237#define DEFAULT_ENCODING_SA 0
 238#define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1))
 239
 240static sci_t make_sci(const u8 *addr, __be16 port)
 241{
 242	sci_t sci;
 243
 244	memcpy(&sci, addr, ETH_ALEN);
 245	memcpy(((char *)&sci) + ETH_ALEN, &port, sizeof(port));
 246
 247	return sci;
 248}
 249
 250static sci_t macsec_active_sci(struct macsec_secy *secy)
 251{
 252	struct macsec_rx_sc *rx_sc = rcu_dereference_bh(secy->rx_sc);
 253
 254	/* Case single RX SC */
 255	if (rx_sc && !rcu_dereference_bh(rx_sc->next))
 256		return (rx_sc->active) ? rx_sc->sci : 0;
 257	/* Case no RX SC or multiple */
 258	else
 259		return 0;
 260}
 261
 262static sci_t macsec_frame_sci(struct macsec_eth_header *hdr, bool sci_present,
 263			      struct macsec_rxh_data *rxd)
 264{
 265	struct macsec_dev *macsec;
 266	sci_t sci = 0;
 267
 268	/* SC = 1 */
 269	if (sci_present) {
 270		memcpy(&sci, hdr->secure_channel_id,
 271		       sizeof(hdr->secure_channel_id));
 272	/* SC = 0; ES = 0 */
 273	} else if ((!(hdr->tci_an & (MACSEC_TCI_ES | MACSEC_TCI_SC))) &&
 274		   (list_is_singular(&rxd->secys))) {
 275		/* Only one SECY should exist on this scenario */
 276		macsec = list_first_or_null_rcu(&rxd->secys, struct macsec_dev,
 277						secys);
 278		if (macsec)
 279			return macsec_active_sci(&macsec->secy);
 280	} else {
 281		sci = make_sci(hdr->eth.h_source, MACSEC_PORT_ES);
 282	}
 283
 284	return sci;
 285}
 286
 287static unsigned int macsec_sectag_len(bool sci_present)
 288{
 289	return MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
 290}
 291
 292static unsigned int macsec_hdr_len(bool sci_present)
 293{
 294	return macsec_sectag_len(sci_present) + ETH_HLEN;
 295}
 296
 297static unsigned int macsec_extra_len(bool sci_present)
 298{
 299	return macsec_sectag_len(sci_present) + sizeof(__be16);
 300}
 301
 302/* Fill SecTAG according to IEEE 802.1AE-2006 10.5.3 */
 303static void macsec_fill_sectag(struct macsec_eth_header *h,
 304			       const struct macsec_secy *secy, u32 pn,
 305			       bool sci_present)
 306{
 307	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
 308
 309	memset(&h->tci_an, 0, macsec_sectag_len(sci_present));
 310	h->eth.h_proto = htons(ETH_P_MACSEC);
 311
 312	if (sci_present) {
 313		h->tci_an |= MACSEC_TCI_SC;
 314		memcpy(&h->secure_channel_id, &secy->sci,
 315		       sizeof(h->secure_channel_id));
 316	} else {
 317		if (tx_sc->end_station)
 318			h->tci_an |= MACSEC_TCI_ES;
 319		if (tx_sc->scb)
 320			h->tci_an |= MACSEC_TCI_SCB;
 321	}
 322
 323	h->packet_number = htonl(pn);
 324
 325	/* with GCM, C/E clear for !encrypt, both set for encrypt */
 326	if (tx_sc->encrypt)
 327		h->tci_an |= MACSEC_TCI_CONFID;
 328	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
 329		h->tci_an |= MACSEC_TCI_C;
 330
 331	h->tci_an |= tx_sc->encoding_sa;
 332}
 333
 334static void macsec_set_shortlen(struct macsec_eth_header *h, size_t data_len)
 335{
 336	if (data_len < MIN_NON_SHORT_LEN)
 337		h->short_length = data_len;
 338}
 339
 340/* Checks if a MACsec interface is being offloaded to an hardware engine */
 341static bool macsec_is_offloaded(struct macsec_dev *macsec)
 342{
 343	if (macsec->offload == MACSEC_OFFLOAD_MAC ||
 344	    macsec->offload == MACSEC_OFFLOAD_PHY)
 345		return true;
 346
 347	return false;
 348}
 349
 350/* Checks if underlying layers implement MACsec offloading functions. */
 351static bool macsec_check_offload(enum macsec_offload offload,
 352				 struct macsec_dev *macsec)
 353{
 354	if (!macsec || !macsec->real_dev)
 355		return false;
 356
 357	if (offload == MACSEC_OFFLOAD_PHY)
 358		return macsec->real_dev->phydev &&
 359		       macsec->real_dev->phydev->macsec_ops;
 360	else if (offload == MACSEC_OFFLOAD_MAC)
 361		return macsec->real_dev->features & NETIF_F_HW_MACSEC &&
 362		       macsec->real_dev->macsec_ops;
 363
 364	return false;
 365}
 366
 367static const struct macsec_ops *__macsec_get_ops(enum macsec_offload offload,
 368						 struct macsec_dev *macsec,
 369						 struct macsec_context *ctx)
 370{
 371	if (ctx) {
 372		memset(ctx, 0, sizeof(*ctx));
 373		ctx->offload = offload;
 374
 375		if (offload == MACSEC_OFFLOAD_PHY)
 376			ctx->phydev = macsec->real_dev->phydev;
 377		else if (offload == MACSEC_OFFLOAD_MAC)
 378			ctx->netdev = macsec->real_dev;
 379	}
 380
 381	if (offload == MACSEC_OFFLOAD_PHY)
 382		return macsec->real_dev->phydev->macsec_ops;
 383	else
 384		return macsec->real_dev->macsec_ops;
 385}
 386
 387/* Returns a pointer to the MACsec ops struct if any and updates the MACsec
 388 * context device reference if provided.
 389 */
 390static const struct macsec_ops *macsec_get_ops(struct macsec_dev *macsec,
 391					       struct macsec_context *ctx)
 392{
 393	if (!macsec_check_offload(macsec->offload, macsec))
 394		return NULL;
 395
 396	return __macsec_get_ops(macsec->offload, macsec, ctx);
 397}
 398
 399/* validate MACsec packet according to IEEE 802.1AE-2018 9.12 */
 400static bool macsec_validate_skb(struct sk_buff *skb, u16 icv_len, bool xpn)
 401{
 402	struct macsec_eth_header *h = (struct macsec_eth_header *)skb->data;
 403	int len = skb->len - 2 * ETH_ALEN;
 404	int extra_len = macsec_extra_len(!!(h->tci_an & MACSEC_TCI_SC)) + icv_len;
 405
 406	/* a) It comprises at least 17 octets */
 407	if (skb->len <= 16)
 408		return false;
 409
 410	/* b) MACsec EtherType: already checked */
 411
 412	/* c) V bit is clear */
 413	if (h->tci_an & MACSEC_TCI_VERSION)
 414		return false;
 415
 416	/* d) ES or SCB => !SC */
 417	if ((h->tci_an & MACSEC_TCI_ES || h->tci_an & MACSEC_TCI_SCB) &&
 418	    (h->tci_an & MACSEC_TCI_SC))
 419		return false;
 420
 421	/* e) Bits 7 and 8 of octet 4 of the SecTAG are clear */
 422	if (h->unused)
 423		return false;
 424
 425	/* rx.pn != 0 if not XPN (figure 10-5 with 802.11AEbw-2013 amendment) */
 426	if (!h->packet_number && !xpn)
 427		return false;
 428
 429	/* length check, f) g) h) i) */
 430	if (h->short_length)
 431		return len == extra_len + h->short_length;
 432	return len >= extra_len + MIN_NON_SHORT_LEN;
 433}
 434
 435#define MACSEC_NEEDED_HEADROOM (macsec_extra_len(true))
 436#define MACSEC_NEEDED_TAILROOM MACSEC_STD_ICV_LEN
 437
 438static void macsec_fill_iv_xpn(unsigned char *iv, ssci_t ssci, u64 pn,
 439			       salt_t salt)
 440{
 441	struct gcm_iv_xpn *gcm_iv = (struct gcm_iv_xpn *)iv;
 442
 443	gcm_iv->ssci = ssci ^ salt.ssci;
 444	gcm_iv->pn = cpu_to_be64(pn) ^ salt.pn;
 445}
 446
 447static void macsec_fill_iv(unsigned char *iv, sci_t sci, u32 pn)
 448{
 449	struct gcm_iv *gcm_iv = (struct gcm_iv *)iv;
 450
 451	gcm_iv->sci = sci;
 452	gcm_iv->pn = htonl(pn);
 453}
 454
 455static struct macsec_eth_header *macsec_ethhdr(struct sk_buff *skb)
 456{
 457	return (struct macsec_eth_header *)skb_mac_header(skb);
 458}
 459
 460static void __macsec_pn_wrapped(struct macsec_secy *secy,
 461				struct macsec_tx_sa *tx_sa)
 462{
 463	pr_debug("PN wrapped, transitioning to !oper\n");
 464	tx_sa->active = false;
 465	if (secy->protect_frames)
 466		secy->operational = false;
 467}
 468
 469void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa)
 470{
 471	spin_lock_bh(&tx_sa->lock);
 472	__macsec_pn_wrapped(secy, tx_sa);
 473	spin_unlock_bh(&tx_sa->lock);
 474}
 475EXPORT_SYMBOL_GPL(macsec_pn_wrapped);
 476
 477static pn_t tx_sa_update_pn(struct macsec_tx_sa *tx_sa,
 478			    struct macsec_secy *secy)
 479{
 480	pn_t pn;
 481
 482	spin_lock_bh(&tx_sa->lock);
 483
 484	pn = tx_sa->next_pn_halves;
 485	if (secy->xpn)
 486		tx_sa->next_pn++;
 487	else
 488		tx_sa->next_pn_halves.lower++;
 489
 490	if (tx_sa->next_pn == 0)
 491		__macsec_pn_wrapped(secy, tx_sa);
 492	spin_unlock_bh(&tx_sa->lock);
 493
 494	return pn;
 495}
 496
 497static void macsec_encrypt_finish(struct sk_buff *skb, struct net_device *dev)
 498{
 499	struct macsec_dev *macsec = netdev_priv(dev);
 500
 501	skb->dev = macsec->real_dev;
 502	skb_reset_mac_header(skb);
 503	skb->protocol = eth_hdr(skb)->h_proto;
 504}
 505
 506static unsigned int macsec_msdu_len(struct sk_buff *skb)
 507{
 508	struct macsec_dev *macsec = macsec_priv(skb->dev);
 509	struct macsec_secy *secy = &macsec->secy;
 510	bool sci_present = macsec_skb_cb(skb)->has_sci;
 511
 512	return skb->len - macsec_hdr_len(sci_present) - secy->icv_len;
 513}
 514
 515static void macsec_count_tx(struct sk_buff *skb, struct macsec_tx_sc *tx_sc,
 516			    struct macsec_tx_sa *tx_sa)
 517{
 518	unsigned int msdu_len = macsec_msdu_len(skb);
 519	struct pcpu_tx_sc_stats *txsc_stats = this_cpu_ptr(tx_sc->stats);
 520
 521	u64_stats_update_begin(&txsc_stats->syncp);
 522	if (tx_sc->encrypt) {
 523		txsc_stats->stats.OutOctetsEncrypted += msdu_len;
 524		txsc_stats->stats.OutPktsEncrypted++;
 525		this_cpu_inc(tx_sa->stats->OutPktsEncrypted);
 526	} else {
 527		txsc_stats->stats.OutOctetsProtected += msdu_len;
 528		txsc_stats->stats.OutPktsProtected++;
 529		this_cpu_inc(tx_sa->stats->OutPktsProtected);
 530	}
 531	u64_stats_update_end(&txsc_stats->syncp);
 532}
 533
 534static void count_tx(struct net_device *dev, int ret, int len)
 535{
 536	if (likely(ret == NET_XMIT_SUCCESS || ret == NET_XMIT_CN))
 537		dev_sw_netstats_tx_add(dev, 1, len);
 538}
 539
 540static void macsec_encrypt_done(void *data, int err)
 541{
 542	struct sk_buff *skb = data;
 543	struct net_device *dev = skb->dev;
 544	struct macsec_dev *macsec = macsec_priv(dev);
 545	struct macsec_tx_sa *sa = macsec_skb_cb(skb)->tx_sa;
 546	int len, ret;
 547
 548	aead_request_free(macsec_skb_cb(skb)->req);
 549
 550	rcu_read_lock_bh();
 551	macsec_count_tx(skb, &macsec->secy.tx_sc, macsec_skb_cb(skb)->tx_sa);
 552	/* packet is encrypted/protected so tx_bytes must be calculated */
 553	len = macsec_msdu_len(skb) + 2 * ETH_ALEN;
 554	macsec_encrypt_finish(skb, dev);
 555	ret = dev_queue_xmit(skb);
 556	count_tx(dev, ret, len);
 557	rcu_read_unlock_bh();
 558
 559	macsec_txsa_put(sa);
 560	dev_put(dev);
 561}
 562
 563static struct aead_request *macsec_alloc_req(struct crypto_aead *tfm,
 564					     unsigned char **iv,
 565					     struct scatterlist **sg,
 566					     int num_frags)
 567{
 568	size_t size, iv_offset, sg_offset;
 569	struct aead_request *req;
 570	void *tmp;
 571
 572	size = sizeof(struct aead_request) + crypto_aead_reqsize(tfm);
 573	iv_offset = size;
 574	size += GCM_AES_IV_LEN;
 575
 576	size = ALIGN(size, __alignof__(struct scatterlist));
 577	sg_offset = size;
 578	size += sizeof(struct scatterlist) * num_frags;
 579
 580	tmp = kmalloc(size, GFP_ATOMIC);
 581	if (!tmp)
 582		return NULL;
 583
 584	*iv = (unsigned char *)(tmp + iv_offset);
 585	*sg = (struct scatterlist *)(tmp + sg_offset);
 586	req = tmp;
 587
 588	aead_request_set_tfm(req, tfm);
 589
 590	return req;
 591}
 592
 593static struct sk_buff *macsec_encrypt(struct sk_buff *skb,
 594				      struct net_device *dev)
 595{
 596	int ret;
 597	struct scatterlist *sg;
 598	struct sk_buff *trailer;
 599	unsigned char *iv;
 600	struct ethhdr *eth;
 601	struct macsec_eth_header *hh;
 602	size_t unprotected_len;
 603	struct aead_request *req;
 604	struct macsec_secy *secy;
 605	struct macsec_tx_sc *tx_sc;
 606	struct macsec_tx_sa *tx_sa;
 607	struct macsec_dev *macsec = macsec_priv(dev);
 608	bool sci_present;
 609	pn_t pn;
 610
 611	secy = &macsec->secy;
 612	tx_sc = &secy->tx_sc;
 613
 614	/* 10.5.1 TX SA assignment */
 615	tx_sa = macsec_txsa_get(tx_sc->sa[tx_sc->encoding_sa]);
 616	if (!tx_sa) {
 617		secy->operational = false;
 618		kfree_skb(skb);
 619		return ERR_PTR(-EINVAL);
 620	}
 621
 622	if (unlikely(skb_headroom(skb) < MACSEC_NEEDED_HEADROOM ||
 623		     skb_tailroom(skb) < MACSEC_NEEDED_TAILROOM)) {
 624		struct sk_buff *nskb = skb_copy_expand(skb,
 625						       MACSEC_NEEDED_HEADROOM,
 626						       MACSEC_NEEDED_TAILROOM,
 627						       GFP_ATOMIC);
 628		if (likely(nskb)) {
 629			consume_skb(skb);
 630			skb = nskb;
 631		} else {
 632			macsec_txsa_put(tx_sa);
 633			kfree_skb(skb);
 634			return ERR_PTR(-ENOMEM);
 635		}
 636	} else {
 637		skb = skb_unshare(skb, GFP_ATOMIC);
 638		if (!skb) {
 639			macsec_txsa_put(tx_sa);
 640			return ERR_PTR(-ENOMEM);
 641		}
 642	}
 643
 644	unprotected_len = skb->len;
 645	eth = eth_hdr(skb);
 646	sci_present = macsec_send_sci(secy);
 647	hh = skb_push(skb, macsec_extra_len(sci_present));
 648	memmove(hh, eth, 2 * ETH_ALEN);
 649
 650	pn = tx_sa_update_pn(tx_sa, secy);
 651	if (pn.full64 == 0) {
 652		macsec_txsa_put(tx_sa);
 653		kfree_skb(skb);
 654		return ERR_PTR(-ENOLINK);
 655	}
 656	macsec_fill_sectag(hh, secy, pn.lower, sci_present);
 657	macsec_set_shortlen(hh, unprotected_len - 2 * ETH_ALEN);
 658
 659	skb_put(skb, secy->icv_len);
 660
 661	if (skb->len - ETH_HLEN > macsec_priv(dev)->real_dev->mtu) {
 662		struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
 663
 664		u64_stats_update_begin(&secy_stats->syncp);
 665		secy_stats->stats.OutPktsTooLong++;
 666		u64_stats_update_end(&secy_stats->syncp);
 667
 668		macsec_txsa_put(tx_sa);
 669		kfree_skb(skb);
 670		return ERR_PTR(-EINVAL);
 671	}
 672
 673	ret = skb_cow_data(skb, 0, &trailer);
 674	if (unlikely(ret < 0)) {
 675		macsec_txsa_put(tx_sa);
 676		kfree_skb(skb);
 677		return ERR_PTR(ret);
 678	}
 679
 680	req = macsec_alloc_req(tx_sa->key.tfm, &iv, &sg, ret);
 681	if (!req) {
 682		macsec_txsa_put(tx_sa);
 683		kfree_skb(skb);
 684		return ERR_PTR(-ENOMEM);
 685	}
 686
 687	if (secy->xpn)
 688		macsec_fill_iv_xpn(iv, tx_sa->ssci, pn.full64, tx_sa->key.salt);
 689	else
 690		macsec_fill_iv(iv, secy->sci, pn.lower);
 691
 692	sg_init_table(sg, ret);
 693	ret = skb_to_sgvec(skb, sg, 0, skb->len);
 694	if (unlikely(ret < 0)) {
 695		aead_request_free(req);
 696		macsec_txsa_put(tx_sa);
 697		kfree_skb(skb);
 698		return ERR_PTR(ret);
 699	}
 700
 701	if (tx_sc->encrypt) {
 702		int len = skb->len - macsec_hdr_len(sci_present) -
 703			  secy->icv_len;
 704		aead_request_set_crypt(req, sg, sg, len, iv);
 705		aead_request_set_ad(req, macsec_hdr_len(sci_present));
 706	} else {
 707		aead_request_set_crypt(req, sg, sg, 0, iv);
 708		aead_request_set_ad(req, skb->len - secy->icv_len);
 709	}
 710
 711	macsec_skb_cb(skb)->req = req;
 712	macsec_skb_cb(skb)->tx_sa = tx_sa;
 713	macsec_skb_cb(skb)->has_sci = sci_present;
 714	aead_request_set_callback(req, 0, macsec_encrypt_done, skb);
 715
 716	dev_hold(skb->dev);
 717	ret = crypto_aead_encrypt(req);
 718	if (ret == -EINPROGRESS) {
 719		return ERR_PTR(ret);
 720	} else if (ret != 0) {
 721		dev_put(skb->dev);
 722		kfree_skb(skb);
 723		aead_request_free(req);
 724		macsec_txsa_put(tx_sa);
 725		return ERR_PTR(-EINVAL);
 726	}
 727
 728	dev_put(skb->dev);
 729	aead_request_free(req);
 730	macsec_txsa_put(tx_sa);
 731
 732	return skb;
 733}
 734
 735static bool macsec_post_decrypt(struct sk_buff *skb, struct macsec_secy *secy, u32 pn)
 736{
 737	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
 738	struct pcpu_rx_sc_stats *rxsc_stats = this_cpu_ptr(rx_sa->sc->stats);
 739	struct macsec_eth_header *hdr = macsec_ethhdr(skb);
 740	u32 lowest_pn = 0;
 741
 742	spin_lock(&rx_sa->lock);
 743	if (rx_sa->next_pn_halves.lower >= secy->replay_window)
 744		lowest_pn = rx_sa->next_pn_halves.lower - secy->replay_window;
 745
 746	/* Now perform replay protection check again
 747	 * (see IEEE 802.1AE-2006 figure 10-5)
 748	 */
 749	if (secy->replay_protect && pn < lowest_pn &&
 750	    (!secy->xpn || pn_same_half(pn, lowest_pn))) {
 751		spin_unlock(&rx_sa->lock);
 752		u64_stats_update_begin(&rxsc_stats->syncp);
 753		rxsc_stats->stats.InPktsLate++;
 754		u64_stats_update_end(&rxsc_stats->syncp);
 755		DEV_STATS_INC(secy->netdev, rx_dropped);
 756		return false;
 757	}
 758
 759	if (secy->validate_frames != MACSEC_VALIDATE_DISABLED) {
 760		unsigned int msdu_len = macsec_msdu_len(skb);
 761		u64_stats_update_begin(&rxsc_stats->syncp);
 762		if (hdr->tci_an & MACSEC_TCI_E)
 763			rxsc_stats->stats.InOctetsDecrypted += msdu_len;
 764		else
 765			rxsc_stats->stats.InOctetsValidated += msdu_len;
 766		u64_stats_update_end(&rxsc_stats->syncp);
 767	}
 768
 769	if (!macsec_skb_cb(skb)->valid) {
 770		spin_unlock(&rx_sa->lock);
 771
 772		/* 10.6.5 */
 773		if (hdr->tci_an & MACSEC_TCI_C ||
 774		    secy->validate_frames == MACSEC_VALIDATE_STRICT) {
 775			u64_stats_update_begin(&rxsc_stats->syncp);
 776			rxsc_stats->stats.InPktsNotValid++;
 777			u64_stats_update_end(&rxsc_stats->syncp);
 778			this_cpu_inc(rx_sa->stats->InPktsNotValid);
 779			DEV_STATS_INC(secy->netdev, rx_errors);
 780			return false;
 781		}
 782
 783		u64_stats_update_begin(&rxsc_stats->syncp);
 784		if (secy->validate_frames == MACSEC_VALIDATE_CHECK) {
 785			rxsc_stats->stats.InPktsInvalid++;
 786			this_cpu_inc(rx_sa->stats->InPktsInvalid);
 787		} else if (pn < lowest_pn) {
 788			rxsc_stats->stats.InPktsDelayed++;
 789		} else {
 790			rxsc_stats->stats.InPktsUnchecked++;
 791		}
 792		u64_stats_update_end(&rxsc_stats->syncp);
 793	} else {
 794		u64_stats_update_begin(&rxsc_stats->syncp);
 795		if (pn < lowest_pn) {
 796			rxsc_stats->stats.InPktsDelayed++;
 797		} else {
 798			rxsc_stats->stats.InPktsOK++;
 799			this_cpu_inc(rx_sa->stats->InPktsOK);
 800		}
 801		u64_stats_update_end(&rxsc_stats->syncp);
 802
 803		// Instead of "pn >=" - to support pn overflow in xpn
 804		if (pn + 1 > rx_sa->next_pn_halves.lower) {
 805			rx_sa->next_pn_halves.lower = pn + 1;
 806		} else if (secy->xpn &&
 807			   !pn_same_half(pn, rx_sa->next_pn_halves.lower)) {
 808			rx_sa->next_pn_halves.upper++;
 809			rx_sa->next_pn_halves.lower = pn + 1;
 810		}
 811
 812		spin_unlock(&rx_sa->lock);
 813	}
 814
 815	return true;
 816}
 817
 818static void macsec_reset_skb(struct sk_buff *skb, struct net_device *dev)
 819{
 820	skb->pkt_type = PACKET_HOST;
 821	skb->protocol = eth_type_trans(skb, dev);
 822
 823	skb_reset_network_header(skb);
 824	if (!skb_transport_header_was_set(skb))
 825		skb_reset_transport_header(skb);
 826	skb_reset_mac_len(skb);
 827}
 828
 829static void macsec_finalize_skb(struct sk_buff *skb, u8 icv_len, u8 hdr_len)
 830{
 831	skb->ip_summed = CHECKSUM_NONE;
 832	memmove(skb->data + hdr_len, skb->data, 2 * ETH_ALEN);
 833	skb_pull(skb, hdr_len);
 834	pskb_trim_unique(skb, skb->len - icv_len);
 835}
 836
 837static void count_rx(struct net_device *dev, int len)
 838{
 839	dev_sw_netstats_rx_add(dev, len);
 840}
 841
 842static void macsec_decrypt_done(void *data, int err)
 843{
 844	struct sk_buff *skb = data;
 845	struct net_device *dev = skb->dev;
 846	struct macsec_dev *macsec = macsec_priv(dev);
 847	struct macsec_rx_sa *rx_sa = macsec_skb_cb(skb)->rx_sa;
 848	struct macsec_rx_sc *rx_sc = rx_sa->sc;
 849	int len;
 850	u32 pn;
 851
 852	aead_request_free(macsec_skb_cb(skb)->req);
 853
 854	if (!err)
 855		macsec_skb_cb(skb)->valid = true;
 856
 857	rcu_read_lock_bh();
 858	pn = ntohl(macsec_ethhdr(skb)->packet_number);
 859	if (!macsec_post_decrypt(skb, &macsec->secy, pn)) {
 860		rcu_read_unlock_bh();
 861		kfree_skb(skb);
 862		goto out;
 863	}
 864
 865	macsec_finalize_skb(skb, macsec->secy.icv_len,
 866			    macsec_extra_len(macsec_skb_cb(skb)->has_sci));
 867	len = skb->len;
 868	macsec_reset_skb(skb, macsec->secy.netdev);
 869
 870	if (gro_cells_receive(&macsec->gro_cells, skb) == NET_RX_SUCCESS)
 871		count_rx(dev, len);
 872
 873	rcu_read_unlock_bh();
 874
 875out:
 876	macsec_rxsa_put(rx_sa);
 877	macsec_rxsc_put(rx_sc);
 878	dev_put(dev);
 879}
 880
 881static struct sk_buff *macsec_decrypt(struct sk_buff *skb,
 882				      struct net_device *dev,
 883				      struct macsec_rx_sa *rx_sa,
 884				      sci_t sci,
 885				      struct macsec_secy *secy)
 886{
 887	int ret;
 888	struct scatterlist *sg;
 889	struct sk_buff *trailer;
 890	unsigned char *iv;
 891	struct aead_request *req;
 892	struct macsec_eth_header *hdr;
 893	u32 hdr_pn;
 894	u16 icv_len = secy->icv_len;
 895
 896	macsec_skb_cb(skb)->valid = false;
 897	skb = skb_share_check(skb, GFP_ATOMIC);
 898	if (!skb)
 899		return ERR_PTR(-ENOMEM);
 900
 901	ret = skb_cow_data(skb, 0, &trailer);
 902	if (unlikely(ret < 0)) {
 903		kfree_skb(skb);
 904		return ERR_PTR(ret);
 905	}
 906	req = macsec_alloc_req(rx_sa->key.tfm, &iv, &sg, ret);
 907	if (!req) {
 908		kfree_skb(skb);
 909		return ERR_PTR(-ENOMEM);
 910	}
 911
 912	hdr = (struct macsec_eth_header *)skb->data;
 913	hdr_pn = ntohl(hdr->packet_number);
 914
 915	if (secy->xpn) {
 916		pn_t recovered_pn = rx_sa->next_pn_halves;
 917
 918		recovered_pn.lower = hdr_pn;
 919		if (hdr_pn < rx_sa->next_pn_halves.lower &&
 920		    !pn_same_half(hdr_pn, rx_sa->next_pn_halves.lower))
 921			recovered_pn.upper++;
 922
 923		macsec_fill_iv_xpn(iv, rx_sa->ssci, recovered_pn.full64,
 924				   rx_sa->key.salt);
 925	} else {
 926		macsec_fill_iv(iv, sci, hdr_pn);
 927	}
 928
 929	sg_init_table(sg, ret);
 930	ret = skb_to_sgvec(skb, sg, 0, skb->len);
 931	if (unlikely(ret < 0)) {
 932		aead_request_free(req);
 933		kfree_skb(skb);
 934		return ERR_PTR(ret);
 935	}
 936
 937	if (hdr->tci_an & MACSEC_TCI_E) {
 938		/* confidentiality: ethernet + macsec header
 939		 * authenticated, encrypted payload
 940		 */
 941		int len = skb->len - macsec_hdr_len(macsec_skb_cb(skb)->has_sci);
 942
 943		aead_request_set_crypt(req, sg, sg, len, iv);
 944		aead_request_set_ad(req, macsec_hdr_len(macsec_skb_cb(skb)->has_sci));
 945		skb = skb_unshare(skb, GFP_ATOMIC);
 946		if (!skb) {
 947			aead_request_free(req);
 948			return ERR_PTR(-ENOMEM);
 949		}
 950	} else {
 951		/* integrity only: all headers + data authenticated */
 952		aead_request_set_crypt(req, sg, sg, icv_len, iv);
 953		aead_request_set_ad(req, skb->len - icv_len);
 954	}
 955
 956	macsec_skb_cb(skb)->req = req;
 957	skb->dev = dev;
 958	aead_request_set_callback(req, 0, macsec_decrypt_done, skb);
 959
 960	dev_hold(dev);
 961	ret = crypto_aead_decrypt(req);
 962	if (ret == -EINPROGRESS) {
 963		return ERR_PTR(ret);
 964	} else if (ret != 0) {
 965		/* decryption/authentication failed
 966		 * 10.6 if validateFrames is disabled, deliver anyway
 967		 */
 968		if (ret != -EBADMSG) {
 969			kfree_skb(skb);
 970			skb = ERR_PTR(ret);
 971		}
 972	} else {
 973		macsec_skb_cb(skb)->valid = true;
 974	}
 975	dev_put(dev);
 976
 977	aead_request_free(req);
 978
 979	return skb;
 980}
 981
 982static struct macsec_rx_sc *find_rx_sc(struct macsec_secy *secy, sci_t sci)
 983{
 984	struct macsec_rx_sc *rx_sc;
 985
 986	for_each_rxsc(secy, rx_sc) {
 987		if (rx_sc->sci == sci)
 988			return rx_sc;
 989	}
 990
 991	return NULL;
 992}
 993
 994static struct macsec_rx_sc *find_rx_sc_rtnl(struct macsec_secy *secy, sci_t sci)
 995{
 996	struct macsec_rx_sc *rx_sc;
 997
 998	for_each_rxsc_rtnl(secy, rx_sc) {
 999		if (rx_sc->sci == sci)
1000			return rx_sc;
1001	}
1002
1003	return NULL;
1004}
1005
1006static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
1007{
1008	/* Deliver to the uncontrolled port by default */
1009	enum rx_handler_result ret = RX_HANDLER_PASS;
1010	struct ethhdr *hdr = eth_hdr(skb);
1011	struct metadata_dst *md_dst;
1012	struct macsec_rxh_data *rxd;
1013	struct macsec_dev *macsec;
1014	bool is_macsec_md_dst;
1015
1016	rcu_read_lock();
1017	rxd = macsec_data_rcu(skb->dev);
1018	md_dst = skb_metadata_dst(skb);
1019	is_macsec_md_dst = md_dst && md_dst->type == METADATA_MACSEC;
1020
1021	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1022		struct sk_buff *nskb;
1023		struct pcpu_secy_stats *secy_stats = this_cpu_ptr(macsec->stats);
1024		struct net_device *ndev = macsec->secy.netdev;
1025
1026		/* If h/w offloading is enabled, HW decodes frames and strips
1027		 * the SecTAG, so we have to deduce which port to deliver to.
1028		 */
1029		if (macsec_is_offloaded(macsec) && netif_running(ndev)) {
1030			const struct macsec_ops *ops;
1031
1032			ops = macsec_get_ops(macsec, NULL);
1033
1034			if (ops->rx_uses_md_dst && !is_macsec_md_dst)
1035				continue;
1036
1037			if (is_macsec_md_dst) {
1038				struct macsec_rx_sc *rx_sc;
1039
1040				/* All drivers that implement MACsec offload
1041				 * support using skb metadata destinations must
1042				 * indicate that they do so.
1043				 */
1044				DEBUG_NET_WARN_ON_ONCE(!ops->rx_uses_md_dst);
1045				rx_sc = find_rx_sc(&macsec->secy,
1046						   md_dst->u.macsec_info.sci);
1047				if (!rx_sc)
1048					continue;
1049				/* device indicated macsec offload occurred */
1050				skb->dev = ndev;
1051				skb->pkt_type = PACKET_HOST;
1052				eth_skb_pkt_type(skb, ndev);
1053				ret = RX_HANDLER_ANOTHER;
1054				goto out;
1055			}
1056
1057			/* This datapath is insecure because it is unable to
1058			 * enforce isolation of broadcast/multicast traffic and
1059			 * unicast traffic with promiscuous mode on the macsec
1060			 * netdev. Since the core stack has no mechanism to
1061			 * check that the hardware did indeed receive MACsec
1062			 * traffic, it is possible that the response handling
1063			 * done by the MACsec port was to a plaintext packet.
1064			 * This violates the MACsec protocol standard.
1065			 */
1066			if (ether_addr_equal_64bits(hdr->h_dest,
1067						    ndev->dev_addr)) {
1068				/* exact match, divert skb to this port */
1069				skb->dev = ndev;
1070				skb->pkt_type = PACKET_HOST;
1071				ret = RX_HANDLER_ANOTHER;
1072				goto out;
1073			} else if (is_multicast_ether_addr_64bits(
1074					   hdr->h_dest)) {
1075				/* multicast frame, deliver on this port too */
1076				nskb = skb_clone(skb, GFP_ATOMIC);
1077				if (!nskb)
1078					break;
1079
1080				nskb->dev = ndev;
1081				eth_skb_pkt_type(nskb, ndev);
1082
1083				__netif_rx(nskb);
1084			} else if (ndev->flags & IFF_PROMISC) {
1085				skb->dev = ndev;
1086				skb->pkt_type = PACKET_HOST;
1087				ret = RX_HANDLER_ANOTHER;
1088				goto out;
1089			}
1090
1091			continue;
1092		}
1093
1094		/* 10.6 If the management control validateFrames is not
1095		 * Strict, frames without a SecTAG are received, counted, and
1096		 * delivered to the Controlled Port
1097		 */
1098		if (macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1099			u64_stats_update_begin(&secy_stats->syncp);
1100			secy_stats->stats.InPktsNoTag++;
1101			u64_stats_update_end(&secy_stats->syncp);
1102			DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1103			continue;
1104		}
1105
1106		/* deliver on this port */
1107		nskb = skb_clone(skb, GFP_ATOMIC);
1108		if (!nskb)
1109			break;
1110
1111		nskb->dev = ndev;
1112
1113		if (__netif_rx(nskb) == NET_RX_SUCCESS) {
1114			u64_stats_update_begin(&secy_stats->syncp);
1115			secy_stats->stats.InPktsUntagged++;
1116			u64_stats_update_end(&secy_stats->syncp);
1117		}
1118	}
1119
1120out:
1121	rcu_read_unlock();
1122	return ret;
1123}
1124
1125static rx_handler_result_t macsec_handle_frame(struct sk_buff **pskb)
1126{
1127	struct sk_buff *skb = *pskb;
1128	struct net_device *dev = skb->dev;
1129	struct macsec_eth_header *hdr;
1130	struct macsec_secy *secy = NULL;
1131	struct macsec_rx_sc *rx_sc;
1132	struct macsec_rx_sa *rx_sa;
1133	struct macsec_rxh_data *rxd;
1134	struct macsec_dev *macsec;
1135	unsigned int len;
1136	sci_t sci = 0;
1137	u32 hdr_pn;
1138	bool cbit;
1139	struct pcpu_rx_sc_stats *rxsc_stats;
1140	struct pcpu_secy_stats *secy_stats;
1141	bool pulled_sci;
1142	int ret;
1143
1144	if (skb_headroom(skb) < ETH_HLEN)
1145		goto drop_direct;
1146
1147	hdr = macsec_ethhdr(skb);
1148	if (hdr->eth.h_proto != htons(ETH_P_MACSEC))
1149		return handle_not_macsec(skb);
1150
1151	skb = skb_unshare(skb, GFP_ATOMIC);
1152	*pskb = skb;
1153	if (!skb)
1154		return RX_HANDLER_CONSUMED;
1155
1156	pulled_sci = pskb_may_pull(skb, macsec_extra_len(true));
1157	if (!pulled_sci) {
1158		if (!pskb_may_pull(skb, macsec_extra_len(false)))
1159			goto drop_direct;
1160	}
1161
1162	hdr = macsec_ethhdr(skb);
1163
1164	/* Frames with a SecTAG that has the TCI E bit set but the C
1165	 * bit clear are discarded, as this reserved encoding is used
1166	 * to identify frames with a SecTAG that are not to be
1167	 * delivered to the Controlled Port.
1168	 */
1169	if ((hdr->tci_an & (MACSEC_TCI_C | MACSEC_TCI_E)) == MACSEC_TCI_E)
1170		return RX_HANDLER_PASS;
1171
1172	/* now, pull the extra length */
1173	if (hdr->tci_an & MACSEC_TCI_SC) {
1174		if (!pulled_sci)
1175			goto drop_direct;
1176	}
1177
1178	/* ethernet header is part of crypto processing */
1179	skb_push(skb, ETH_HLEN);
1180
1181	macsec_skb_cb(skb)->has_sci = !!(hdr->tci_an & MACSEC_TCI_SC);
1182	macsec_skb_cb(skb)->assoc_num = hdr->tci_an & MACSEC_AN_MASK;
1183
1184	rcu_read_lock();
1185	rxd = macsec_data_rcu(skb->dev);
1186
1187	sci = macsec_frame_sci(hdr, macsec_skb_cb(skb)->has_sci, rxd);
1188	if (!sci)
1189		goto drop_nosc;
1190
1191	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1192		struct macsec_rx_sc *sc = find_rx_sc(&macsec->secy, sci);
1193
1194		sc = sc ? macsec_rxsc_get(sc) : NULL;
1195
1196		if (sc) {
1197			secy = &macsec->secy;
1198			rx_sc = sc;
1199			break;
1200		}
1201	}
1202
1203	if (!secy)
1204		goto nosci;
1205
1206	dev = secy->netdev;
1207	macsec = macsec_priv(dev);
1208	secy_stats = this_cpu_ptr(macsec->stats);
1209	rxsc_stats = this_cpu_ptr(rx_sc->stats);
1210
1211	if (!macsec_validate_skb(skb, secy->icv_len, secy->xpn)) {
1212		u64_stats_update_begin(&secy_stats->syncp);
1213		secy_stats->stats.InPktsBadTag++;
1214		u64_stats_update_end(&secy_stats->syncp);
1215		DEV_STATS_INC(secy->netdev, rx_errors);
1216		goto drop_nosa;
1217	}
1218
1219	rx_sa = macsec_rxsa_get(rx_sc->sa[macsec_skb_cb(skb)->assoc_num]);
1220	if (!rx_sa) {
1221		/* 10.6.1 if the SA is not in use */
1222
1223		/* If validateFrames is Strict or the C bit in the
1224		 * SecTAG is set, discard
1225		 */
1226		if (hdr->tci_an & MACSEC_TCI_C ||
1227		    secy->validate_frames == MACSEC_VALIDATE_STRICT) {
1228			u64_stats_update_begin(&rxsc_stats->syncp);
1229			rxsc_stats->stats.InPktsNotUsingSA++;
1230			u64_stats_update_end(&rxsc_stats->syncp);
1231			DEV_STATS_INC(secy->netdev, rx_errors);
1232			goto drop_nosa;
1233		}
1234
1235		/* not Strict, the frame (with the SecTAG and ICV
1236		 * removed) is delivered to the Controlled Port.
1237		 */
1238		u64_stats_update_begin(&rxsc_stats->syncp);
1239		rxsc_stats->stats.InPktsUnusedSA++;
1240		u64_stats_update_end(&rxsc_stats->syncp);
1241		goto deliver;
1242	}
1243
1244	/* First, PN check to avoid decrypting obviously wrong packets */
1245	hdr_pn = ntohl(hdr->packet_number);
1246	if (secy->replay_protect) {
1247		bool late;
1248
1249		spin_lock(&rx_sa->lock);
1250		late = rx_sa->next_pn_halves.lower >= secy->replay_window &&
1251		       hdr_pn < (rx_sa->next_pn_halves.lower - secy->replay_window);
1252
1253		if (secy->xpn)
1254			late = late && pn_same_half(rx_sa->next_pn_halves.lower, hdr_pn);
1255		spin_unlock(&rx_sa->lock);
1256
1257		if (late) {
1258			u64_stats_update_begin(&rxsc_stats->syncp);
1259			rxsc_stats->stats.InPktsLate++;
1260			u64_stats_update_end(&rxsc_stats->syncp);
1261			DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1262			goto drop;
1263		}
1264	}
1265
1266	macsec_skb_cb(skb)->rx_sa = rx_sa;
1267
1268	/* Disabled && !changed text => skip validation */
1269	if (hdr->tci_an & MACSEC_TCI_C ||
1270	    secy->validate_frames != MACSEC_VALIDATE_DISABLED)
1271		skb = macsec_decrypt(skb, dev, rx_sa, sci, secy);
1272
1273	if (IS_ERR(skb)) {
1274		/* the decrypt callback needs the reference */
1275		if (PTR_ERR(skb) != -EINPROGRESS) {
1276			macsec_rxsa_put(rx_sa);
1277			macsec_rxsc_put(rx_sc);
1278		}
1279		rcu_read_unlock();
1280		*pskb = NULL;
1281		return RX_HANDLER_CONSUMED;
1282	}
1283
1284	if (!macsec_post_decrypt(skb, secy, hdr_pn))
1285		goto drop;
1286
1287deliver:
1288	macsec_finalize_skb(skb, secy->icv_len,
1289			    macsec_extra_len(macsec_skb_cb(skb)->has_sci));
1290	len = skb->len;
1291	macsec_reset_skb(skb, secy->netdev);
1292
1293	if (rx_sa)
1294		macsec_rxsa_put(rx_sa);
1295	macsec_rxsc_put(rx_sc);
1296
1297	skb_orphan(skb);
1298	ret = gro_cells_receive(&macsec->gro_cells, skb);
1299	if (ret == NET_RX_SUCCESS)
1300		count_rx(dev, len);
1301	else
1302		DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1303
1304	rcu_read_unlock();
1305
1306	*pskb = NULL;
1307	return RX_HANDLER_CONSUMED;
1308
1309drop:
1310	macsec_rxsa_put(rx_sa);
1311drop_nosa:
1312	macsec_rxsc_put(rx_sc);
1313drop_nosc:
1314	rcu_read_unlock();
1315drop_direct:
1316	kfree_skb(skb);
1317	*pskb = NULL;
1318	return RX_HANDLER_CONSUMED;
1319
1320nosci:
1321	/* 10.6.1 if the SC is not found */
1322	cbit = !!(hdr->tci_an & MACSEC_TCI_C);
1323	if (!cbit)
1324		macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN,
1325				    macsec_extra_len(macsec_skb_cb(skb)->has_sci));
1326
1327	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
1328		struct sk_buff *nskb;
1329
1330		secy_stats = this_cpu_ptr(macsec->stats);
1331
1332		/* If validateFrames is Strict or the C bit in the
1333		 * SecTAG is set, discard
1334		 */
1335		if (cbit ||
1336		    macsec->secy.validate_frames == MACSEC_VALIDATE_STRICT) {
1337			u64_stats_update_begin(&secy_stats->syncp);
1338			secy_stats->stats.InPktsNoSCI++;
1339			u64_stats_update_end(&secy_stats->syncp);
1340			DEV_STATS_INC(macsec->secy.netdev, rx_errors);
1341			continue;
1342		}
1343
1344		/* not strict, the frame (with the SecTAG and ICV
1345		 * removed) is delivered to the Controlled Port.
1346		 */
1347		nskb = skb_clone(skb, GFP_ATOMIC);
1348		if (!nskb)
1349			break;
1350
1351		macsec_reset_skb(nskb, macsec->secy.netdev);
1352
1353		ret = __netif_rx(nskb);
1354		if (ret == NET_RX_SUCCESS) {
1355			u64_stats_update_begin(&secy_stats->syncp);
1356			secy_stats->stats.InPktsUnknownSCI++;
1357			u64_stats_update_end(&secy_stats->syncp);
1358		} else {
1359			DEV_STATS_INC(macsec->secy.netdev, rx_dropped);
1360		}
1361	}
1362
1363	rcu_read_unlock();
1364	*pskb = skb;
1365	return RX_HANDLER_PASS;
1366}
1367
1368static struct crypto_aead *macsec_alloc_tfm(char *key, int key_len, int icv_len)
1369{
1370	struct crypto_aead *tfm;
1371	int ret;
1372
1373	tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
1374
1375	if (IS_ERR(tfm))
1376		return tfm;
1377
1378	ret = crypto_aead_setkey(tfm, key, key_len);
1379	if (ret < 0)
1380		goto fail;
1381
1382	ret = crypto_aead_setauthsize(tfm, icv_len);
1383	if (ret < 0)
1384		goto fail;
1385
1386	return tfm;
1387fail:
1388	crypto_free_aead(tfm);
1389	return ERR_PTR(ret);
1390}
1391
1392static int init_rx_sa(struct macsec_rx_sa *rx_sa, char *sak, int key_len,
1393		      int icv_len)
1394{
1395	rx_sa->stats = alloc_percpu(struct macsec_rx_sa_stats);
1396	if (!rx_sa->stats)
1397		return -ENOMEM;
1398
1399	rx_sa->key.tfm = macsec_alloc_tfm(sak, key_len, icv_len);
1400	if (IS_ERR(rx_sa->key.tfm)) {
1401		free_percpu(rx_sa->stats);
1402		return PTR_ERR(rx_sa->key.tfm);
1403	}
1404
1405	rx_sa->ssci = MACSEC_UNDEF_SSCI;
1406	rx_sa->active = false;
1407	rx_sa->next_pn = 1;
1408	refcount_set(&rx_sa->refcnt, 1);
1409	spin_lock_init(&rx_sa->lock);
1410
1411	return 0;
1412}
1413
1414static void clear_rx_sa(struct macsec_rx_sa *rx_sa)
1415{
1416	rx_sa->active = false;
1417
1418	macsec_rxsa_put(rx_sa);
1419}
1420
1421static void free_rx_sc(struct macsec_rx_sc *rx_sc)
1422{
1423	int i;
1424
1425	for (i = 0; i < MACSEC_NUM_AN; i++) {
1426		struct macsec_rx_sa *sa = rtnl_dereference(rx_sc->sa[i]);
1427
1428		RCU_INIT_POINTER(rx_sc->sa[i], NULL);
1429		if (sa)
1430			clear_rx_sa(sa);
1431	}
1432
1433	macsec_rxsc_put(rx_sc);
1434}
1435
1436static struct macsec_rx_sc *del_rx_sc(struct macsec_secy *secy, sci_t sci)
1437{
1438	struct macsec_rx_sc *rx_sc, __rcu **rx_scp;
1439
1440	for (rx_scp = &secy->rx_sc, rx_sc = rtnl_dereference(*rx_scp);
1441	     rx_sc;
1442	     rx_scp = &rx_sc->next, rx_sc = rtnl_dereference(*rx_scp)) {
1443		if (rx_sc->sci == sci) {
1444			if (rx_sc->active)
1445				secy->n_rx_sc--;
1446			rcu_assign_pointer(*rx_scp, rx_sc->next);
1447			return rx_sc;
1448		}
1449	}
1450
1451	return NULL;
1452}
1453
1454static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci,
1455					 bool active)
1456{
1457	struct macsec_rx_sc *rx_sc;
1458	struct macsec_dev *macsec;
1459	struct net_device *real_dev = macsec_priv(dev)->real_dev;
1460	struct macsec_rxh_data *rxd = macsec_data_rtnl(real_dev);
1461	struct macsec_secy *secy;
1462
1463	list_for_each_entry(macsec, &rxd->secys, secys) {
1464		if (find_rx_sc_rtnl(&macsec->secy, sci))
1465			return ERR_PTR(-EEXIST);
1466	}
1467
1468	rx_sc = kzalloc(sizeof(*rx_sc), GFP_KERNEL);
1469	if (!rx_sc)
1470		return ERR_PTR(-ENOMEM);
1471
1472	rx_sc->stats = netdev_alloc_pcpu_stats(struct pcpu_rx_sc_stats);
1473	if (!rx_sc->stats) {
1474		kfree(rx_sc);
1475		return ERR_PTR(-ENOMEM);
1476	}
1477
1478	rx_sc->sci = sci;
1479	rx_sc->active = active;
1480	refcount_set(&rx_sc->refcnt, 1);
1481
1482	secy = &macsec_priv(dev)->secy;
1483	rcu_assign_pointer(rx_sc->next, secy->rx_sc);
1484	rcu_assign_pointer(secy->rx_sc, rx_sc);
1485
1486	if (rx_sc->active)
1487		secy->n_rx_sc++;
1488
1489	return rx_sc;
1490}
1491
1492static int init_tx_sa(struct macsec_tx_sa *tx_sa, char *sak, int key_len,
1493		      int icv_len)
1494{
1495	tx_sa->stats = alloc_percpu(struct macsec_tx_sa_stats);
1496	if (!tx_sa->stats)
1497		return -ENOMEM;
1498
1499	tx_sa->key.tfm = macsec_alloc_tfm(sak, key_len, icv_len);
1500	if (IS_ERR(tx_sa->key.tfm)) {
1501		free_percpu(tx_sa->stats);
1502		return PTR_ERR(tx_sa->key.tfm);
1503	}
1504
1505	tx_sa->ssci = MACSEC_UNDEF_SSCI;
1506	tx_sa->active = false;
1507	refcount_set(&tx_sa->refcnt, 1);
1508	spin_lock_init(&tx_sa->lock);
1509
1510	return 0;
1511}
1512
1513static void clear_tx_sa(struct macsec_tx_sa *tx_sa)
1514{
1515	tx_sa->active = false;
1516
1517	macsec_txsa_put(tx_sa);
1518}
1519
1520static struct genl_family macsec_fam;
1521
1522static struct net_device *get_dev_from_nl(struct net *net,
1523					  struct nlattr **attrs)
1524{
1525	int ifindex = nla_get_u32(attrs[MACSEC_ATTR_IFINDEX]);
1526	struct net_device *dev;
1527
1528	dev = __dev_get_by_index(net, ifindex);
1529	if (!dev)
1530		return ERR_PTR(-ENODEV);
1531
1532	if (!netif_is_macsec(dev))
1533		return ERR_PTR(-ENODEV);
1534
1535	return dev;
1536}
1537
1538static enum macsec_offload nla_get_offload(const struct nlattr *nla)
1539{
1540	return (__force enum macsec_offload)nla_get_u8(nla);
1541}
1542
1543static sci_t nla_get_sci(const struct nlattr *nla)
1544{
1545	return (__force sci_t)nla_get_u64(nla);
1546}
1547
1548static int nla_put_sci(struct sk_buff *skb, int attrtype, sci_t value,
1549		       int padattr)
1550{
1551	return nla_put_u64_64bit(skb, attrtype, (__force u64)value, padattr);
1552}
1553
1554static ssci_t nla_get_ssci(const struct nlattr *nla)
1555{
1556	return (__force ssci_t)nla_get_u32(nla);
1557}
1558
1559static int nla_put_ssci(struct sk_buff *skb, int attrtype, ssci_t value)
1560{
1561	return nla_put_u32(skb, attrtype, (__force u64)value);
1562}
1563
1564static struct macsec_tx_sa *get_txsa_from_nl(struct net *net,
1565					     struct nlattr **attrs,
1566					     struct nlattr **tb_sa,
1567					     struct net_device **devp,
1568					     struct macsec_secy **secyp,
1569					     struct macsec_tx_sc **scp,
1570					     u8 *assoc_num)
1571{
1572	struct net_device *dev;
1573	struct macsec_secy *secy;
1574	struct macsec_tx_sc *tx_sc;
1575	struct macsec_tx_sa *tx_sa;
1576
1577	if (!tb_sa[MACSEC_SA_ATTR_AN])
1578		return ERR_PTR(-EINVAL);
1579
1580	*assoc_num = nla_get_u8(tb_sa[MACSEC_SA_ATTR_AN]);
1581
1582	dev = get_dev_from_nl(net, attrs);
1583	if (IS_ERR(dev))
1584		return ERR_CAST(dev);
1585
1586	secy = &macsec_priv(dev)->secy;
1587	tx_sc = &secy->tx_sc;
1588
1589	tx_sa = rtnl_dereference(tx_sc->sa[*assoc_num]);
1590	if (!tx_sa)
1591		return ERR_PTR(-ENODEV);
1592
1593	*devp = dev;
1594	*scp = tx_sc;
1595	*secyp = secy;
1596	return tx_sa;
1597}
1598
1599static struct macsec_rx_sc *get_rxsc_from_nl(struct net *net,
1600					     struct nlattr **attrs,
1601					     struct nlattr **tb_rxsc,
1602					     struct net_device **devp,
1603					     struct macsec_secy **secyp)
1604{
1605	struct net_device *dev;
1606	struct macsec_secy *secy;
1607	struct macsec_rx_sc *rx_sc;
1608	sci_t sci;
1609
1610	dev = get_dev_from_nl(net, attrs);
1611	if (IS_ERR(dev))
1612		return ERR_CAST(dev);
1613
1614	secy = &macsec_priv(dev)->secy;
1615
1616	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
1617		return ERR_PTR(-EINVAL);
1618
1619	sci = nla_get_sci(tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1620	rx_sc = find_rx_sc_rtnl(secy, sci);
1621	if (!rx_sc)
1622		return ERR_PTR(-ENODEV);
1623
1624	*secyp = secy;
1625	*devp = dev;
1626
1627	return rx_sc;
1628}
1629
1630static struct macsec_rx_sa *get_rxsa_from_nl(struct net *net,
1631					     struct nlattr **attrs,
1632					     struct nlattr **tb_rxsc,
1633					     struct nlattr **tb_sa,
1634					     struct net_device **devp,
1635					     struct macsec_secy **secyp,
1636					     struct macsec_rx_sc **scp,
1637					     u8 *assoc_num)
1638{
1639	struct macsec_rx_sc *rx_sc;
1640	struct macsec_rx_sa *rx_sa;
1641
1642	if (!tb_sa[MACSEC_SA_ATTR_AN])
1643		return ERR_PTR(-EINVAL);
1644
1645	*assoc_num = nla_get_u8(tb_sa[MACSEC_SA_ATTR_AN]);
1646
1647	rx_sc = get_rxsc_from_nl(net, attrs, tb_rxsc, devp, secyp);
1648	if (IS_ERR(rx_sc))
1649		return ERR_CAST(rx_sc);
1650
1651	rx_sa = rtnl_dereference(rx_sc->sa[*assoc_num]);
1652	if (!rx_sa)
1653		return ERR_PTR(-ENODEV);
1654
1655	*scp = rx_sc;
1656	return rx_sa;
1657}
1658
1659static const struct nla_policy macsec_genl_policy[NUM_MACSEC_ATTR] = {
1660	[MACSEC_ATTR_IFINDEX] = { .type = NLA_U32 },
1661	[MACSEC_ATTR_RXSC_CONFIG] = { .type = NLA_NESTED },
1662	[MACSEC_ATTR_SA_CONFIG] = { .type = NLA_NESTED },
1663	[MACSEC_ATTR_OFFLOAD] = { .type = NLA_NESTED },
1664};
1665
1666static const struct nla_policy macsec_genl_rxsc_policy[NUM_MACSEC_RXSC_ATTR] = {
1667	[MACSEC_RXSC_ATTR_SCI] = { .type = NLA_U64 },
1668	[MACSEC_RXSC_ATTR_ACTIVE] = NLA_POLICY_MAX(NLA_U8, 1),
1669};
1670
1671static const struct nla_policy macsec_genl_sa_policy[NUM_MACSEC_SA_ATTR] = {
1672	[MACSEC_SA_ATTR_AN] = NLA_POLICY_MAX(NLA_U8, MACSEC_NUM_AN - 1),
1673	[MACSEC_SA_ATTR_ACTIVE] = NLA_POLICY_MAX(NLA_U8, 1),
1674	[MACSEC_SA_ATTR_PN] = NLA_POLICY_MIN(NLA_UINT, 1),
1675	[MACSEC_SA_ATTR_KEYID] = NLA_POLICY_EXACT_LEN(MACSEC_KEYID_LEN),
1676	[MACSEC_SA_ATTR_KEY] = NLA_POLICY_MAX_LEN(MACSEC_MAX_KEY_LEN),
1677	[MACSEC_SA_ATTR_SSCI] = { .type = NLA_U32 },
1678	[MACSEC_SA_ATTR_SALT] = NLA_POLICY_EXACT_LEN(MACSEC_SALT_LEN),
1679};
1680
1681static const struct nla_policy macsec_genl_offload_policy[NUM_MACSEC_OFFLOAD_ATTR] = {
1682	[MACSEC_OFFLOAD_ATTR_TYPE] = NLA_POLICY_MAX(NLA_U8, MACSEC_OFFLOAD_MAX),
1683};
1684
1685/* Offloads an operation to a device driver */
1686static int macsec_offload(int (* const func)(struct macsec_context *),
1687			  struct macsec_context *ctx)
1688{
1689	int ret;
1690
1691	if (unlikely(!func))
1692		return 0;
1693
1694	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1695		mutex_lock(&ctx->phydev->lock);
1696
1697	ret = (*func)(ctx);
1698
1699	if (ctx->offload == MACSEC_OFFLOAD_PHY)
1700		mutex_unlock(&ctx->phydev->lock);
1701
1702	return ret;
1703}
1704
1705static int parse_sa_config(struct nlattr **attrs, struct nlattr **tb_sa)
1706{
1707	if (!attrs[MACSEC_ATTR_SA_CONFIG])
1708		return -EINVAL;
1709
1710	if (nla_parse_nested_deprecated(tb_sa, MACSEC_SA_ATTR_MAX, attrs[MACSEC_ATTR_SA_CONFIG], macsec_genl_sa_policy, NULL))
1711		return -EINVAL;
1712
1713	return 0;
1714}
1715
1716static int parse_rxsc_config(struct nlattr **attrs, struct nlattr **tb_rxsc)
1717{
1718	if (!attrs[MACSEC_ATTR_RXSC_CONFIG])
1719		return -EINVAL;
1720
1721	if (nla_parse_nested_deprecated(tb_rxsc, MACSEC_RXSC_ATTR_MAX, attrs[MACSEC_ATTR_RXSC_CONFIG], macsec_genl_rxsc_policy, NULL))
1722		return -EINVAL;
1723
1724	return 0;
1725}
1726
1727static bool validate_add_rxsa(struct nlattr **attrs)
1728{
1729	if (!attrs[MACSEC_SA_ATTR_AN] ||
1730	    !attrs[MACSEC_SA_ATTR_KEY] ||
1731	    !attrs[MACSEC_SA_ATTR_KEYID])
1732		return false;
1733
1734	return true;
1735}
1736
1737static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
1738{
1739	struct net_device *dev;
1740	struct nlattr **attrs = info->attrs;
1741	struct macsec_secy *secy;
1742	struct macsec_rx_sc *rx_sc;
1743	struct macsec_rx_sa *rx_sa;
1744	unsigned char assoc_num;
1745	int pn_len;
1746	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1747	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1748	int err;
1749
1750	if (!attrs[MACSEC_ATTR_IFINDEX])
1751		return -EINVAL;
1752
1753	if (parse_sa_config(attrs, tb_sa))
1754		return -EINVAL;
1755
1756	if (parse_rxsc_config(attrs, tb_rxsc))
1757		return -EINVAL;
1758
1759	if (!validate_add_rxsa(tb_sa))
1760		return -EINVAL;
1761
1762	rtnl_lock();
1763	rx_sc = get_rxsc_from_nl(genl_info_net(info), attrs, tb_rxsc, &dev, &secy);
1764	if (IS_ERR(rx_sc)) {
1765		rtnl_unlock();
1766		return PTR_ERR(rx_sc);
1767	}
1768
1769	assoc_num = nla_get_u8(tb_sa[MACSEC_SA_ATTR_AN]);
1770
1771	if (nla_len(tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1772		pr_notice("macsec: nl: add_rxsa: bad key length: %d != %d\n",
1773			  nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
1774		rtnl_unlock();
1775		return -EINVAL;
1776	}
1777
1778	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
1779	if (tb_sa[MACSEC_SA_ATTR_PN] &&
1780	    nla_len(tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
1781		pr_notice("macsec: nl: add_rxsa: bad pn length: %d != %d\n",
1782			  nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
1783		rtnl_unlock();
1784		return -EINVAL;
1785	}
1786
1787	if (secy->xpn) {
1788		if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
1789			rtnl_unlock();
1790			return -EINVAL;
1791		}
1792	}
1793
1794	rx_sa = rtnl_dereference(rx_sc->sa[assoc_num]);
1795	if (rx_sa) {
1796		rtnl_unlock();
1797		return -EBUSY;
1798	}
1799
1800	rx_sa = kmalloc(sizeof(*rx_sa), GFP_KERNEL);
1801	if (!rx_sa) {
1802		rtnl_unlock();
1803		return -ENOMEM;
1804	}
1805
1806	err = init_rx_sa(rx_sa, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
1807			 secy->key_len, secy->icv_len);
1808	if (err < 0) {
1809		kfree(rx_sa);
1810		rtnl_unlock();
1811		return err;
1812	}
1813
1814	if (tb_sa[MACSEC_SA_ATTR_PN]) {
1815		spin_lock_bh(&rx_sa->lock);
1816		rx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]);
1817		spin_unlock_bh(&rx_sa->lock);
1818	}
1819
1820	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
1821		rx_sa->active = !!nla_get_u8(tb_sa[MACSEC_SA_ATTR_ACTIVE]);
1822
1823	rx_sa->sc = rx_sc;
1824
1825	if (secy->xpn) {
1826		rx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
1827		nla_memcpy(rx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
1828			   MACSEC_SALT_LEN);
1829	}
1830
1831	/* If h/w offloading is available, propagate to the device */
1832	if (macsec_is_offloaded(netdev_priv(dev))) {
1833		const struct macsec_ops *ops;
1834		struct macsec_context ctx;
1835
1836		ops = macsec_get_ops(netdev_priv(dev), &ctx);
1837		if (!ops) {
1838			err = -EOPNOTSUPP;
1839			goto cleanup;
1840		}
1841
1842		ctx.sa.assoc_num = assoc_num;
1843		ctx.sa.rx_sa = rx_sa;
1844		ctx.secy = secy;
1845		memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
1846		       secy->key_len);
1847
1848		err = macsec_offload(ops->mdo_add_rxsa, &ctx);
1849		memzero_explicit(ctx.sa.key, secy->key_len);
1850		if (err)
1851			goto cleanup;
1852	}
1853
1854	nla_memcpy(rx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
1855	rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa);
1856
1857	rtnl_unlock();
1858
1859	return 0;
1860
1861cleanup:
1862	macsec_rxsa_put(rx_sa);
1863	rtnl_unlock();
1864	return err;
1865}
1866
1867static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info)
1868{
1869	struct net_device *dev;
1870	sci_t sci = MACSEC_UNDEF_SCI;
1871	struct nlattr **attrs = info->attrs;
1872	struct macsec_rx_sc *rx_sc;
1873	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
1874	struct macsec_secy *secy;
1875	bool active = true;
1876	int ret;
1877
1878	if (!attrs[MACSEC_ATTR_IFINDEX])
1879		return -EINVAL;
1880
1881	if (parse_rxsc_config(attrs, tb_rxsc))
1882		return -EINVAL;
1883
1884	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
1885		return -EINVAL;
1886
1887	rtnl_lock();
1888	dev = get_dev_from_nl(genl_info_net(info), attrs);
1889	if (IS_ERR(dev)) {
1890		rtnl_unlock();
1891		return PTR_ERR(dev);
1892	}
1893
1894	secy = &macsec_priv(dev)->secy;
1895	sci = nla_get_sci(tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
1896
1897	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE])
1898		active = nla_get_u8(tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
1899
1900	rx_sc = create_rx_sc(dev, sci, active);
1901	if (IS_ERR(rx_sc)) {
1902		rtnl_unlock();
1903		return PTR_ERR(rx_sc);
1904	}
1905
1906	if (macsec_is_offloaded(netdev_priv(dev))) {
1907		const struct macsec_ops *ops;
1908		struct macsec_context ctx;
1909
1910		ops = macsec_get_ops(netdev_priv(dev), &ctx);
1911		if (!ops) {
1912			ret = -EOPNOTSUPP;
1913			goto cleanup;
1914		}
1915
1916		ctx.rx_sc = rx_sc;
1917		ctx.secy = secy;
1918
1919		ret = macsec_offload(ops->mdo_add_rxsc, &ctx);
1920		if (ret)
1921			goto cleanup;
1922	}
1923
1924	rtnl_unlock();
1925
1926	return 0;
1927
1928cleanup:
1929	del_rx_sc(secy, sci);
1930	free_rx_sc(rx_sc);
1931	rtnl_unlock();
1932	return ret;
1933}
1934
1935static bool validate_add_txsa(struct nlattr **attrs)
1936{
1937	if (!attrs[MACSEC_SA_ATTR_AN] ||
1938	    !attrs[MACSEC_SA_ATTR_PN] ||
1939	    !attrs[MACSEC_SA_ATTR_KEY] ||
1940	    !attrs[MACSEC_SA_ATTR_KEYID])
1941		return false;
1942
1943	return true;
1944}
1945
1946static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
1947{
1948	struct net_device *dev;
1949	struct nlattr **attrs = info->attrs;
1950	struct macsec_secy *secy;
1951	struct macsec_tx_sc *tx_sc;
1952	struct macsec_tx_sa *tx_sa;
1953	unsigned char assoc_num;
1954	int pn_len;
1955	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
1956	bool was_operational;
1957	int err;
1958
1959	if (!attrs[MACSEC_ATTR_IFINDEX])
1960		return -EINVAL;
1961
1962	if (parse_sa_config(attrs, tb_sa))
1963		return -EINVAL;
1964
1965	if (!validate_add_txsa(tb_sa))
1966		return -EINVAL;
1967
1968	rtnl_lock();
1969	dev = get_dev_from_nl(genl_info_net(info), attrs);
1970	if (IS_ERR(dev)) {
1971		rtnl_unlock();
1972		return PTR_ERR(dev);
1973	}
1974
1975	secy = &macsec_priv(dev)->secy;
1976	tx_sc = &secy->tx_sc;
1977
1978	assoc_num = nla_get_u8(tb_sa[MACSEC_SA_ATTR_AN]);
1979
1980	if (nla_len(tb_sa[MACSEC_SA_ATTR_KEY]) != secy->key_len) {
1981		pr_notice("macsec: nl: add_txsa: bad key length: %d != %d\n",
1982			  nla_len(tb_sa[MACSEC_SA_ATTR_KEY]), secy->key_len);
1983		rtnl_unlock();
1984		return -EINVAL;
1985	}
1986
1987	pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
1988	if (nla_len(tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
1989		pr_notice("macsec: nl: add_txsa: bad pn length: %d != %d\n",
1990			  nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
1991		rtnl_unlock();
1992		return -EINVAL;
1993	}
1994
1995	if (secy->xpn) {
1996		if (!tb_sa[MACSEC_SA_ATTR_SSCI] || !tb_sa[MACSEC_SA_ATTR_SALT]) {
1997			rtnl_unlock();
1998			return -EINVAL;
1999		}
2000	}
2001
2002	tx_sa = rtnl_dereference(tx_sc->sa[assoc_num]);
2003	if (tx_sa) {
2004		rtnl_unlock();
2005		return -EBUSY;
2006	}
2007
2008	tx_sa = kmalloc(sizeof(*tx_sa), GFP_KERNEL);
2009	if (!tx_sa) {
2010		rtnl_unlock();
2011		return -ENOMEM;
2012	}
2013
2014	err = init_tx_sa(tx_sa, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
2015			 secy->key_len, secy->icv_len);
2016	if (err < 0) {
2017		kfree(tx_sa);
2018		rtnl_unlock();
2019		return err;
2020	}
2021
2022	spin_lock_bh(&tx_sa->lock);
2023	tx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]);
2024	spin_unlock_bh(&tx_sa->lock);
2025
2026	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2027		tx_sa->active = !!nla_get_u8(tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2028
2029	was_operational = secy->operational;
2030	if (assoc_num == tx_sc->encoding_sa && tx_sa->active)
2031		secy->operational = true;
2032
2033	if (secy->xpn) {
2034		tx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
2035		nla_memcpy(tx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
2036			   MACSEC_SALT_LEN);
2037	}
2038
2039	/* If h/w offloading is available, propagate to the device */
2040	if (macsec_is_offloaded(netdev_priv(dev))) {
2041		const struct macsec_ops *ops;
2042		struct macsec_context ctx;
2043
2044		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2045		if (!ops) {
2046			err = -EOPNOTSUPP;
2047			goto cleanup;
2048		}
2049
2050		ctx.sa.assoc_num = assoc_num;
2051		ctx.sa.tx_sa = tx_sa;
2052		ctx.secy = secy;
2053		memcpy(ctx.sa.key, nla_data(tb_sa[MACSEC_SA_ATTR_KEY]),
2054		       secy->key_len);
2055
2056		err = macsec_offload(ops->mdo_add_txsa, &ctx);
2057		memzero_explicit(ctx.sa.key, secy->key_len);
2058		if (err)
2059			goto cleanup;
2060	}
2061
2062	nla_memcpy(tx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
2063	rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa);
2064
2065	rtnl_unlock();
2066
2067	return 0;
2068
2069cleanup:
2070	secy->operational = was_operational;
2071	macsec_txsa_put(tx_sa);
2072	rtnl_unlock();
2073	return err;
2074}
2075
2076static int macsec_del_rxsa(struct sk_buff *skb, struct genl_info *info)
2077{
2078	struct nlattr **attrs = info->attrs;
2079	struct net_device *dev;
2080	struct macsec_secy *secy;
2081	struct macsec_rx_sc *rx_sc;
2082	struct macsec_rx_sa *rx_sa;
2083	u8 assoc_num;
2084	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2085	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2086	int ret;
2087
2088	if (!attrs[MACSEC_ATTR_IFINDEX])
2089		return -EINVAL;
2090
2091	if (parse_sa_config(attrs, tb_sa))
2092		return -EINVAL;
2093
2094	if (parse_rxsc_config(attrs, tb_rxsc))
2095		return -EINVAL;
2096
2097	rtnl_lock();
2098	rx_sa = get_rxsa_from_nl(genl_info_net(info), attrs, tb_rxsc, tb_sa,
2099				 &dev, &secy, &rx_sc, &assoc_num);
2100	if (IS_ERR(rx_sa)) {
2101		rtnl_unlock();
2102		return PTR_ERR(rx_sa);
2103	}
2104
2105	if (rx_sa->active) {
2106		rtnl_unlock();
2107		return -EBUSY;
2108	}
2109
2110	/* If h/w offloading is available, propagate to the device */
2111	if (macsec_is_offloaded(netdev_priv(dev))) {
2112		const struct macsec_ops *ops;
2113		struct macsec_context ctx;
2114
2115		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2116		if (!ops) {
2117			ret = -EOPNOTSUPP;
2118			goto cleanup;
2119		}
2120
2121		ctx.sa.assoc_num = assoc_num;
2122		ctx.sa.rx_sa = rx_sa;
2123		ctx.secy = secy;
2124
2125		ret = macsec_offload(ops->mdo_del_rxsa, &ctx);
2126		if (ret)
2127			goto cleanup;
2128	}
2129
2130	RCU_INIT_POINTER(rx_sc->sa[assoc_num], NULL);
2131	clear_rx_sa(rx_sa);
2132
2133	rtnl_unlock();
2134
2135	return 0;
2136
2137cleanup:
2138	rtnl_unlock();
2139	return ret;
2140}
2141
2142static int macsec_del_rxsc(struct sk_buff *skb, struct genl_info *info)
2143{
2144	struct nlattr **attrs = info->attrs;
2145	struct net_device *dev;
2146	struct macsec_secy *secy;
2147	struct macsec_rx_sc *rx_sc;
2148	sci_t sci;
2149	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2150	int ret;
2151
2152	if (!attrs[MACSEC_ATTR_IFINDEX])
2153		return -EINVAL;
2154
2155	if (parse_rxsc_config(attrs, tb_rxsc))
2156		return -EINVAL;
2157
2158	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
2159		return -EINVAL;
2160
2161	rtnl_lock();
2162	dev = get_dev_from_nl(genl_info_net(info), info->attrs);
2163	if (IS_ERR(dev)) {
2164		rtnl_unlock();
2165		return PTR_ERR(dev);
2166	}
2167
2168	secy = &macsec_priv(dev)->secy;
2169	sci = nla_get_sci(tb_rxsc[MACSEC_RXSC_ATTR_SCI]);
2170
2171	rx_sc = del_rx_sc(secy, sci);
2172	if (!rx_sc) {
2173		rtnl_unlock();
2174		return -ENODEV;
2175	}
2176
2177	/* If h/w offloading is available, propagate to the device */
2178	if (macsec_is_offloaded(netdev_priv(dev))) {
2179		const struct macsec_ops *ops;
2180		struct macsec_context ctx;
2181
2182		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2183		if (!ops) {
2184			ret = -EOPNOTSUPP;
2185			goto cleanup;
2186		}
2187
2188		ctx.rx_sc = rx_sc;
2189		ctx.secy = secy;
2190		ret = macsec_offload(ops->mdo_del_rxsc, &ctx);
2191		if (ret)
2192			goto cleanup;
2193	}
2194
2195	free_rx_sc(rx_sc);
2196	rtnl_unlock();
2197
2198	return 0;
2199
2200cleanup:
2201	rtnl_unlock();
2202	return ret;
2203}
2204
2205static int macsec_del_txsa(struct sk_buff *skb, struct genl_info *info)
2206{
2207	struct nlattr **attrs = info->attrs;
2208	struct net_device *dev;
2209	struct macsec_secy *secy;
2210	struct macsec_tx_sc *tx_sc;
2211	struct macsec_tx_sa *tx_sa;
2212	u8 assoc_num;
2213	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2214	int ret;
2215
2216	if (!attrs[MACSEC_ATTR_IFINDEX])
2217		return -EINVAL;
2218
2219	if (parse_sa_config(attrs, tb_sa))
2220		return -EINVAL;
2221
2222	rtnl_lock();
2223	tx_sa = get_txsa_from_nl(genl_info_net(info), attrs, tb_sa,
2224				 &dev, &secy, &tx_sc, &assoc_num);
2225	if (IS_ERR(tx_sa)) {
2226		rtnl_unlock();
2227		return PTR_ERR(tx_sa);
2228	}
2229
2230	if (tx_sa->active) {
2231		rtnl_unlock();
2232		return -EBUSY;
2233	}
2234
2235	/* If h/w offloading is available, propagate to the device */
2236	if (macsec_is_offloaded(netdev_priv(dev))) {
2237		const struct macsec_ops *ops;
2238		struct macsec_context ctx;
2239
2240		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2241		if (!ops) {
2242			ret = -EOPNOTSUPP;
2243			goto cleanup;
2244		}
2245
2246		ctx.sa.assoc_num = assoc_num;
2247		ctx.sa.tx_sa = tx_sa;
2248		ctx.secy = secy;
2249
2250		ret = macsec_offload(ops->mdo_del_txsa, &ctx);
2251		if (ret)
2252			goto cleanup;
2253	}
2254
2255	RCU_INIT_POINTER(tx_sc->sa[assoc_num], NULL);
2256	clear_tx_sa(tx_sa);
2257
2258	rtnl_unlock();
2259
2260	return 0;
2261
2262cleanup:
2263	rtnl_unlock();
2264	return ret;
2265}
2266
2267static bool validate_upd_sa(struct nlattr **attrs)
2268{
2269	if (!attrs[MACSEC_SA_ATTR_AN] ||
2270	    attrs[MACSEC_SA_ATTR_KEY] ||
2271	    attrs[MACSEC_SA_ATTR_KEYID] ||
2272	    attrs[MACSEC_SA_ATTR_SSCI] ||
2273	    attrs[MACSEC_SA_ATTR_SALT])
2274		return false;
2275
2276	return true;
2277}
2278
2279static int macsec_upd_txsa(struct sk_buff *skb, struct genl_info *info)
2280{
2281	struct nlattr **attrs = info->attrs;
2282	struct net_device *dev;
2283	struct macsec_secy *secy;
2284	struct macsec_tx_sc *tx_sc;
2285	struct macsec_tx_sa *tx_sa;
2286	u8 assoc_num;
2287	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2288	bool was_operational, was_active;
2289	pn_t prev_pn;
2290	int ret = 0;
2291
2292	prev_pn.full64 = 0;
2293
2294	if (!attrs[MACSEC_ATTR_IFINDEX])
2295		return -EINVAL;
2296
2297	if (parse_sa_config(attrs, tb_sa))
2298		return -EINVAL;
2299
2300	if (!validate_upd_sa(tb_sa))
2301		return -EINVAL;
2302
2303	rtnl_lock();
2304	tx_sa = get_txsa_from_nl(genl_info_net(info), attrs, tb_sa,
2305				 &dev, &secy, &tx_sc, &assoc_num);
2306	if (IS_ERR(tx_sa)) {
2307		rtnl_unlock();
2308		return PTR_ERR(tx_sa);
2309	}
2310
2311	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2312		int pn_len;
2313
2314		pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2315		if (nla_len(tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2316			pr_notice("macsec: nl: upd_txsa: bad pn length: %d != %d\n",
2317				  nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2318			rtnl_unlock();
2319			return -EINVAL;
2320		}
2321
2322		spin_lock_bh(&tx_sa->lock);
2323		prev_pn = tx_sa->next_pn_halves;
2324		tx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]);
2325		spin_unlock_bh(&tx_sa->lock);
2326	}
2327
2328	was_active = tx_sa->active;
2329	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2330		tx_sa->active = nla_get_u8(tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2331
2332	was_operational = secy->operational;
2333	if (assoc_num == tx_sc->encoding_sa)
2334		secy->operational = tx_sa->active;
2335
2336	/* If h/w offloading is available, propagate to the device */
2337	if (macsec_is_offloaded(netdev_priv(dev))) {
2338		const struct macsec_ops *ops;
2339		struct macsec_context ctx;
2340
2341		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2342		if (!ops) {
2343			ret = -EOPNOTSUPP;
2344			goto cleanup;
2345		}
2346
2347		ctx.sa.assoc_num = assoc_num;
2348		ctx.sa.tx_sa = tx_sa;
2349		ctx.sa.update_pn = !!prev_pn.full64;
2350		ctx.secy = secy;
2351
2352		ret = macsec_offload(ops->mdo_upd_txsa, &ctx);
2353		if (ret)
2354			goto cleanup;
2355	}
2356
2357	rtnl_unlock();
2358
2359	return 0;
2360
2361cleanup:
2362	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2363		spin_lock_bh(&tx_sa->lock);
2364		tx_sa->next_pn_halves = prev_pn;
2365		spin_unlock_bh(&tx_sa->lock);
2366	}
2367	tx_sa->active = was_active;
2368	secy->operational = was_operational;
2369	rtnl_unlock();
2370	return ret;
2371}
2372
2373static int macsec_upd_rxsa(struct sk_buff *skb, struct genl_info *info)
2374{
2375	struct nlattr **attrs = info->attrs;
2376	struct net_device *dev;
2377	struct macsec_secy *secy;
2378	struct macsec_rx_sc *rx_sc;
2379	struct macsec_rx_sa *rx_sa;
2380	u8 assoc_num;
2381	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2382	struct nlattr *tb_sa[MACSEC_SA_ATTR_MAX + 1];
2383	bool was_active;
2384	pn_t prev_pn;
2385	int ret = 0;
2386
2387	prev_pn.full64 = 0;
2388
2389	if (!attrs[MACSEC_ATTR_IFINDEX])
2390		return -EINVAL;
2391
2392	if (parse_rxsc_config(attrs, tb_rxsc))
2393		return -EINVAL;
2394
2395	if (parse_sa_config(attrs, tb_sa))
2396		return -EINVAL;
2397
2398	if (!validate_upd_sa(tb_sa))
2399		return -EINVAL;
2400
2401	rtnl_lock();
2402	rx_sa = get_rxsa_from_nl(genl_info_net(info), attrs, tb_rxsc, tb_sa,
2403				 &dev, &secy, &rx_sc, &assoc_num);
2404	if (IS_ERR(rx_sa)) {
2405		rtnl_unlock();
2406		return PTR_ERR(rx_sa);
2407	}
2408
2409	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2410		int pn_len;
2411
2412		pn_len = secy->xpn ? MACSEC_XPN_PN_LEN : MACSEC_DEFAULT_PN_LEN;
2413		if (nla_len(tb_sa[MACSEC_SA_ATTR_PN]) != pn_len) {
2414			pr_notice("macsec: nl: upd_rxsa: bad pn length: %d != %d\n",
2415				  nla_len(tb_sa[MACSEC_SA_ATTR_PN]), pn_len);
2416			rtnl_unlock();
2417			return -EINVAL;
2418		}
2419
2420		spin_lock_bh(&rx_sa->lock);
2421		prev_pn = rx_sa->next_pn_halves;
2422		rx_sa->next_pn = nla_get_uint(tb_sa[MACSEC_SA_ATTR_PN]);
2423		spin_unlock_bh(&rx_sa->lock);
2424	}
2425
2426	was_active = rx_sa->active;
2427	if (tb_sa[MACSEC_SA_ATTR_ACTIVE])
2428		rx_sa->active = nla_get_u8(tb_sa[MACSEC_SA_ATTR_ACTIVE]);
2429
2430	/* If h/w offloading is available, propagate to the device */
2431	if (macsec_is_offloaded(netdev_priv(dev))) {
2432		const struct macsec_ops *ops;
2433		struct macsec_context ctx;
2434
2435		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2436		if (!ops) {
2437			ret = -EOPNOTSUPP;
2438			goto cleanup;
2439		}
2440
2441		ctx.sa.assoc_num = assoc_num;
2442		ctx.sa.rx_sa = rx_sa;
2443		ctx.sa.update_pn = !!prev_pn.full64;
2444		ctx.secy = secy;
2445
2446		ret = macsec_offload(ops->mdo_upd_rxsa, &ctx);
2447		if (ret)
2448			goto cleanup;
2449	}
2450
2451	rtnl_unlock();
2452	return 0;
2453
2454cleanup:
2455	if (tb_sa[MACSEC_SA_ATTR_PN]) {
2456		spin_lock_bh(&rx_sa->lock);
2457		rx_sa->next_pn_halves = prev_pn;
2458		spin_unlock_bh(&rx_sa->lock);
2459	}
2460	rx_sa->active = was_active;
2461	rtnl_unlock();
2462	return ret;
2463}
2464
2465static int macsec_upd_rxsc(struct sk_buff *skb, struct genl_info *info)
2466{
2467	struct nlattr **attrs = info->attrs;
2468	struct net_device *dev;
2469	struct macsec_secy *secy;
2470	struct macsec_rx_sc *rx_sc;
2471	struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1];
2472	unsigned int prev_n_rx_sc;
2473	bool was_active;
2474	int ret;
2475
2476	if (!attrs[MACSEC_ATTR_IFINDEX])
2477		return -EINVAL;
2478
2479	if (parse_rxsc_config(attrs, tb_rxsc))
2480		return -EINVAL;
2481
2482	if (!tb_rxsc[MACSEC_RXSC_ATTR_SCI])
2483		return -EINVAL;
2484
2485	rtnl_lock();
2486	rx_sc = get_rxsc_from_nl(genl_info_net(info), attrs, tb_rxsc, &dev, &secy);
2487	if (IS_ERR(rx_sc)) {
2488		rtnl_unlock();
2489		return PTR_ERR(rx_sc);
2490	}
2491
2492	was_active = rx_sc->active;
2493	prev_n_rx_sc = secy->n_rx_sc;
2494	if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) {
2495		bool new = !!nla_get_u8(tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]);
2496
2497		if (rx_sc->active != new)
2498			secy->n_rx_sc += new ? 1 : -1;
2499
2500		rx_sc->active = new;
2501	}
2502
2503	/* If h/w offloading is available, propagate to the device */
2504	if (macsec_is_offloaded(netdev_priv(dev))) {
2505		const struct macsec_ops *ops;
2506		struct macsec_context ctx;
2507
2508		ops = macsec_get_ops(netdev_priv(dev), &ctx);
2509		if (!ops) {
2510			ret = -EOPNOTSUPP;
2511			goto cleanup;
2512		}
2513
2514		ctx.rx_sc = rx_sc;
2515		ctx.secy = secy;
2516
2517		ret = macsec_offload(ops->mdo_upd_rxsc, &ctx);
2518		if (ret)
2519			goto cleanup;
2520	}
2521
2522	rtnl_unlock();
2523
2524	return 0;
2525
2526cleanup:
2527	secy->n_rx_sc = prev_n_rx_sc;
2528	rx_sc->active = was_active;
2529	rtnl_unlock();
2530	return ret;
2531}
2532
2533static bool macsec_is_configured(struct macsec_dev *macsec)
2534{
2535	struct macsec_secy *secy = &macsec->secy;
2536	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2537	int i;
2538
2539	if (secy->rx_sc)
2540		return true;
2541
2542	for (i = 0; i < MACSEC_NUM_AN; i++)
2543		if (tx_sc->sa[i])
2544			return true;
2545
2546	return false;
2547}
2548
2549static bool macsec_needs_tx_tag(struct macsec_dev *macsec,
2550				const struct macsec_ops *ops)
2551{
2552	return macsec->offload == MACSEC_OFFLOAD_PHY &&
2553		ops->mdo_insert_tx_tag;
2554}
2555
2556static void macsec_set_head_tail_room(struct net_device *dev)
2557{
2558	struct macsec_dev *macsec = macsec_priv(dev);
2559	struct net_device *real_dev = macsec->real_dev;
2560	int needed_headroom, needed_tailroom;
2561	const struct macsec_ops *ops;
2562
2563	ops = macsec_get_ops(macsec, NULL);
2564	if (ops) {
2565		needed_headroom = ops->needed_headroom;
2566		needed_tailroom = ops->needed_tailroom;
2567	} else {
2568		needed_headroom = MACSEC_NEEDED_HEADROOM;
2569		needed_tailroom = MACSEC_NEEDED_TAILROOM;
2570	}
2571
2572	dev->needed_headroom = real_dev->needed_headroom + needed_headroom;
2573	dev->needed_tailroom = real_dev->needed_tailroom + needed_tailroom;
2574}
2575
2576static void macsec_inherit_tso_max(struct net_device *dev)
2577{
2578	struct macsec_dev *macsec = macsec_priv(dev);
2579
2580	/* if macsec is offloaded, we need to follow the lower
2581	 * device's capabilities. otherwise, we can ignore them.
2582	 */
2583	if (macsec_is_offloaded(macsec))
2584		netif_inherit_tso_max(dev, macsec->real_dev);
2585}
2586
2587static int macsec_update_offload(struct net_device *dev, enum macsec_offload offload)
2588{
2589	enum macsec_offload prev_offload;
2590	const struct macsec_ops *ops;
2591	struct macsec_context ctx;
2592	struct macsec_dev *macsec;
2593	int ret = 0;
2594
2595	macsec = macsec_priv(dev);
2596
2597	/* Check if the offloading mode is supported by the underlying layers */
2598	if (offload != MACSEC_OFFLOAD_OFF &&
2599	    !macsec_check_offload(offload, macsec))
2600		return -EOPNOTSUPP;
2601
2602	/* Check if the net device is busy. */
2603	if (netif_running(dev))
2604		return -EBUSY;
2605
2606	/* Check if the device already has rules configured: we do not support
2607	 * rules migration.
2608	 */
2609	if (macsec_is_configured(macsec))
2610		return -EBUSY;
2611
2612	prev_offload = macsec->offload;
2613
2614	ops = __macsec_get_ops(offload == MACSEC_OFFLOAD_OFF ? prev_offload : offload,
2615			       macsec, &ctx);
2616	if (!ops)
2617		return -EOPNOTSUPP;
2618
2619	macsec->offload = offload;
2620
2621	ctx.secy = &macsec->secy;
2622	ret = offload == MACSEC_OFFLOAD_OFF ? macsec_offload(ops->mdo_del_secy, &ctx)
2623					    : macsec_offload(ops->mdo_add_secy, &ctx);
2624	if (ret) {
2625		macsec->offload = prev_offload;
2626		return ret;
2627	}
2628
2629	macsec_set_head_tail_room(dev);
2630	macsec->insert_tx_tag = macsec_needs_tx_tag(macsec, ops);
2631
2632	macsec_inherit_tso_max(dev);
2633
2634	netdev_update_features(dev);
2635
2636	return ret;
2637}
2638
2639static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info)
2640{
2641	struct nlattr *tb_offload[MACSEC_OFFLOAD_ATTR_MAX + 1];
2642	struct nlattr **attrs = info->attrs;
2643	enum macsec_offload offload;
2644	struct macsec_dev *macsec;
2645	struct net_device *dev;
2646	int ret = 0;
2647
2648	if (!attrs[MACSEC_ATTR_IFINDEX])
2649		return -EINVAL;
2650
2651	if (!attrs[MACSEC_ATTR_OFFLOAD])
2652		return -EINVAL;
2653
2654	if (nla_parse_nested_deprecated(tb_offload, MACSEC_OFFLOAD_ATTR_MAX,
2655					attrs[MACSEC_ATTR_OFFLOAD],
2656					macsec_genl_offload_policy, NULL))
2657		return -EINVAL;
2658
2659	rtnl_lock();
2660
2661	dev = get_dev_from_nl(genl_info_net(info), attrs);
2662	if (IS_ERR(dev)) {
2663		ret = PTR_ERR(dev);
2664		goto out;
2665	}
2666	macsec = macsec_priv(dev);
2667
2668	if (!tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]) {
2669		ret = -EINVAL;
2670		goto out;
2671	}
2672
2673	offload = nla_get_u8(tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]);
2674
2675	if (macsec->offload != offload)
2676		ret = macsec_update_offload(dev, offload);
2677out:
2678	rtnl_unlock();
2679	return ret;
2680}
2681
2682static void get_tx_sa_stats(struct net_device *dev, int an,
2683			    struct macsec_tx_sa *tx_sa,
2684			    struct macsec_tx_sa_stats *sum)
2685{
2686	struct macsec_dev *macsec = macsec_priv(dev);
2687	int cpu;
2688
2689	/* If h/w offloading is available, propagate to the device */
2690	if (macsec_is_offloaded(macsec)) {
2691		const struct macsec_ops *ops;
2692		struct macsec_context ctx;
2693
2694		ops = macsec_get_ops(macsec, &ctx);
2695		if (ops) {
2696			ctx.sa.assoc_num = an;
2697			ctx.sa.tx_sa = tx_sa;
2698			ctx.stats.tx_sa_stats = sum;
2699			ctx.secy = &macsec_priv(dev)->secy;
2700			macsec_offload(ops->mdo_get_tx_sa_stats, &ctx);
2701		}
2702		return;
2703	}
2704
2705	for_each_possible_cpu(cpu) {
2706		const struct macsec_tx_sa_stats *stats =
2707			per_cpu_ptr(tx_sa->stats, cpu);
2708
2709		sum->OutPktsProtected += stats->OutPktsProtected;
2710		sum->OutPktsEncrypted += stats->OutPktsEncrypted;
2711	}
2712}
2713
2714static int copy_tx_sa_stats(struct sk_buff *skb, struct macsec_tx_sa_stats *sum)
2715{
2716	if (nla_put_u32(skb, MACSEC_SA_STATS_ATTR_OUT_PKTS_PROTECTED,
2717			sum->OutPktsProtected) ||
2718	    nla_put_u32(skb, MACSEC_SA_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2719			sum->OutPktsEncrypted))
2720		return -EMSGSIZE;
2721
2722	return 0;
2723}
2724
2725static void get_rx_sa_stats(struct net_device *dev,
2726			    struct macsec_rx_sc *rx_sc, int an,
2727			    struct macsec_rx_sa *rx_sa,
2728			    struct macsec_rx_sa_stats *sum)
2729{
2730	struct macsec_dev *macsec = macsec_priv(dev);
2731	int cpu;
2732
2733	/* If h/w offloading is available, propagate to the device */
2734	if (macsec_is_offloaded(macsec)) {
2735		const struct macsec_ops *ops;
2736		struct macsec_context ctx;
2737
2738		ops = macsec_get_ops(macsec, &ctx);
2739		if (ops) {
2740			ctx.sa.assoc_num = an;
2741			ctx.sa.rx_sa = rx_sa;
2742			ctx.stats.rx_sa_stats = sum;
2743			ctx.secy = &macsec_priv(dev)->secy;
2744			ctx.rx_sc = rx_sc;
2745			macsec_offload(ops->mdo_get_rx_sa_stats, &ctx);
2746		}
2747		return;
2748	}
2749
2750	for_each_possible_cpu(cpu) {
2751		const struct macsec_rx_sa_stats *stats =
2752			per_cpu_ptr(rx_sa->stats, cpu);
2753
2754		sum->InPktsOK         += stats->InPktsOK;
2755		sum->InPktsInvalid    += stats->InPktsInvalid;
2756		sum->InPktsNotValid   += stats->InPktsNotValid;
2757		sum->InPktsNotUsingSA += stats->InPktsNotUsingSA;
2758		sum->InPktsUnusedSA   += stats->InPktsUnusedSA;
2759	}
2760}
2761
2762static int copy_rx_sa_stats(struct sk_buff *skb,
2763			    struct macsec_rx_sa_stats *sum)
2764{
2765	if (nla_put_u32(skb, MACSEC_SA_STATS_ATTR_IN_PKTS_OK, sum->InPktsOK) ||
2766	    nla_put_u32(skb, MACSEC_SA_STATS_ATTR_IN_PKTS_INVALID,
2767			sum->InPktsInvalid) ||
2768	    nla_put_u32(skb, MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_VALID,
2769			sum->InPktsNotValid) ||
2770	    nla_put_u32(skb, MACSEC_SA_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2771			sum->InPktsNotUsingSA) ||
2772	    nla_put_u32(skb, MACSEC_SA_STATS_ATTR_IN_PKTS_UNUSED_SA,
2773			sum->InPktsUnusedSA))
2774		return -EMSGSIZE;
2775
2776	return 0;
2777}
2778
2779static void get_rx_sc_stats(struct net_device *dev,
2780			    struct macsec_rx_sc *rx_sc,
2781			    struct macsec_rx_sc_stats *sum)
2782{
2783	struct macsec_dev *macsec = macsec_priv(dev);
2784	int cpu;
2785
2786	/* If h/w offloading is available, propagate to the device */
2787	if (macsec_is_offloaded(macsec)) {
2788		const struct macsec_ops *ops;
2789		struct macsec_context ctx;
2790
2791		ops = macsec_get_ops(macsec, &ctx);
2792		if (ops) {
2793			ctx.stats.rx_sc_stats = sum;
2794			ctx.secy = &macsec_priv(dev)->secy;
2795			ctx.rx_sc = rx_sc;
2796			macsec_offload(ops->mdo_get_rx_sc_stats, &ctx);
2797		}
2798		return;
2799	}
2800
2801	for_each_possible_cpu(cpu) {
2802		const struct pcpu_rx_sc_stats *stats;
2803		struct macsec_rx_sc_stats tmp;
2804		unsigned int start;
2805
2806		stats = per_cpu_ptr(rx_sc->stats, cpu);
2807		do {
2808			start = u64_stats_fetch_begin(&stats->syncp);
2809			memcpy(&tmp, &stats->stats, sizeof(tmp));
2810		} while (u64_stats_fetch_retry(&stats->syncp, start));
2811
2812		sum->InOctetsValidated += tmp.InOctetsValidated;
2813		sum->InOctetsDecrypted += tmp.InOctetsDecrypted;
2814		sum->InPktsUnchecked   += tmp.InPktsUnchecked;
2815		sum->InPktsDelayed     += tmp.InPktsDelayed;
2816		sum->InPktsOK          += tmp.InPktsOK;
2817		sum->InPktsInvalid     += tmp.InPktsInvalid;
2818		sum->InPktsLate        += tmp.InPktsLate;
2819		sum->InPktsNotValid    += tmp.InPktsNotValid;
2820		sum->InPktsNotUsingSA  += tmp.InPktsNotUsingSA;
2821		sum->InPktsUnusedSA    += tmp.InPktsUnusedSA;
2822	}
2823}
2824
2825static int copy_rx_sc_stats(struct sk_buff *skb, struct macsec_rx_sc_stats *sum)
2826{
2827	if (nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_OCTETS_VALIDATED,
2828			      sum->InOctetsValidated,
2829			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2830	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_OCTETS_DECRYPTED,
2831			      sum->InOctetsDecrypted,
2832			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2833	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNCHECKED,
2834			      sum->InPktsUnchecked,
2835			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2836	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_DELAYED,
2837			      sum->InPktsDelayed,
2838			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2839	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_OK,
2840			      sum->InPktsOK,
2841			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2842	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_INVALID,
2843			      sum->InPktsInvalid,
2844			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2845	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_LATE,
2846			      sum->InPktsLate,
2847			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2848	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_VALID,
2849			      sum->InPktsNotValid,
2850			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2851	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_NOT_USING_SA,
2852			      sum->InPktsNotUsingSA,
2853			      MACSEC_RXSC_STATS_ATTR_PAD) ||
2854	    nla_put_u64_64bit(skb, MACSEC_RXSC_STATS_ATTR_IN_PKTS_UNUSED_SA,
2855			      sum->InPktsUnusedSA,
2856			      MACSEC_RXSC_STATS_ATTR_PAD))
2857		return -EMSGSIZE;
2858
2859	return 0;
2860}
2861
2862static void get_tx_sc_stats(struct net_device *dev,
2863			    struct macsec_tx_sc_stats *sum)
2864{
2865	struct macsec_dev *macsec = macsec_priv(dev);
2866	int cpu;
2867
2868	/* If h/w offloading is available, propagate to the device */
2869	if (macsec_is_offloaded(macsec)) {
2870		const struct macsec_ops *ops;
2871		struct macsec_context ctx;
2872
2873		ops = macsec_get_ops(macsec, &ctx);
2874		if (ops) {
2875			ctx.stats.tx_sc_stats = sum;
2876			ctx.secy = &macsec_priv(dev)->secy;
2877			macsec_offload(ops->mdo_get_tx_sc_stats, &ctx);
2878		}
2879		return;
2880	}
2881
2882	for_each_possible_cpu(cpu) {
2883		const struct pcpu_tx_sc_stats *stats;
2884		struct macsec_tx_sc_stats tmp;
2885		unsigned int start;
2886
2887		stats = per_cpu_ptr(macsec_priv(dev)->secy.tx_sc.stats, cpu);
2888		do {
2889			start = u64_stats_fetch_begin(&stats->syncp);
2890			memcpy(&tmp, &stats->stats, sizeof(tmp));
2891		} while (u64_stats_fetch_retry(&stats->syncp, start));
2892
2893		sum->OutPktsProtected   += tmp.OutPktsProtected;
2894		sum->OutPktsEncrypted   += tmp.OutPktsEncrypted;
2895		sum->OutOctetsProtected += tmp.OutOctetsProtected;
2896		sum->OutOctetsEncrypted += tmp.OutOctetsEncrypted;
2897	}
2898}
2899
2900static int copy_tx_sc_stats(struct sk_buff *skb, struct macsec_tx_sc_stats *sum)
2901{
2902	if (nla_put_u64_64bit(skb, MACSEC_TXSC_STATS_ATTR_OUT_PKTS_PROTECTED,
2903			      sum->OutPktsProtected,
2904			      MACSEC_TXSC_STATS_ATTR_PAD) ||
2905	    nla_put_u64_64bit(skb, MACSEC_TXSC_STATS_ATTR_OUT_PKTS_ENCRYPTED,
2906			      sum->OutPktsEncrypted,
2907			      MACSEC_TXSC_STATS_ATTR_PAD) ||
2908	    nla_put_u64_64bit(skb, MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_PROTECTED,
2909			      sum->OutOctetsProtected,
2910			      MACSEC_TXSC_STATS_ATTR_PAD) ||
2911	    nla_put_u64_64bit(skb, MACSEC_TXSC_STATS_ATTR_OUT_OCTETS_ENCRYPTED,
2912			      sum->OutOctetsEncrypted,
2913			      MACSEC_TXSC_STATS_ATTR_PAD))
2914		return -EMSGSIZE;
2915
2916	return 0;
2917}
2918
2919static void get_secy_stats(struct net_device *dev, struct macsec_dev_stats *sum)
2920{
2921	struct macsec_dev *macsec = macsec_priv(dev);
2922	int cpu;
2923
2924	/* If h/w offloading is available, propagate to the device */
2925	if (macsec_is_offloaded(macsec)) {
2926		const struct macsec_ops *ops;
2927		struct macsec_context ctx;
2928
2929		ops = macsec_get_ops(macsec, &ctx);
2930		if (ops) {
2931			ctx.stats.dev_stats = sum;
2932			ctx.secy = &macsec_priv(dev)->secy;
2933			macsec_offload(ops->mdo_get_dev_stats, &ctx);
2934		}
2935		return;
2936	}
2937
2938	for_each_possible_cpu(cpu) {
2939		const struct pcpu_secy_stats *stats;
2940		struct macsec_dev_stats tmp;
2941		unsigned int start;
2942
2943		stats = per_cpu_ptr(macsec_priv(dev)->stats, cpu);
2944		do {
2945			start = u64_stats_fetch_begin(&stats->syncp);
2946			memcpy(&tmp, &stats->stats, sizeof(tmp));
2947		} while (u64_stats_fetch_retry(&stats->syncp, start));
2948
2949		sum->OutPktsUntagged  += tmp.OutPktsUntagged;
2950		sum->InPktsUntagged   += tmp.InPktsUntagged;
2951		sum->OutPktsTooLong   += tmp.OutPktsTooLong;
2952		sum->InPktsNoTag      += tmp.InPktsNoTag;
2953		sum->InPktsBadTag     += tmp.InPktsBadTag;
2954		sum->InPktsUnknownSCI += tmp.InPktsUnknownSCI;
2955		sum->InPktsNoSCI      += tmp.InPktsNoSCI;
2956		sum->InPktsOverrun    += tmp.InPktsOverrun;
2957	}
2958}
2959
2960static int copy_secy_stats(struct sk_buff *skb, struct macsec_dev_stats *sum)
2961{
2962	if (nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_OUT_PKTS_UNTAGGED,
2963			      sum->OutPktsUntagged,
2964			      MACSEC_SECY_STATS_ATTR_PAD) ||
2965	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_UNTAGGED,
2966			      sum->InPktsUntagged,
2967			      MACSEC_SECY_STATS_ATTR_PAD) ||
2968	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_OUT_PKTS_TOO_LONG,
2969			      sum->OutPktsTooLong,
2970			      MACSEC_SECY_STATS_ATTR_PAD) ||
2971	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_TAG,
2972			      sum->InPktsNoTag,
2973			      MACSEC_SECY_STATS_ATTR_PAD) ||
2974	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_BAD_TAG,
2975			      sum->InPktsBadTag,
2976			      MACSEC_SECY_STATS_ATTR_PAD) ||
2977	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_UNKNOWN_SCI,
2978			      sum->InPktsUnknownSCI,
2979			      MACSEC_SECY_STATS_ATTR_PAD) ||
2980	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_NO_SCI,
2981			      sum->InPktsNoSCI,
2982			      MACSEC_SECY_STATS_ATTR_PAD) ||
2983	    nla_put_u64_64bit(skb, MACSEC_SECY_STATS_ATTR_IN_PKTS_OVERRUN,
2984			      sum->InPktsOverrun,
2985			      MACSEC_SECY_STATS_ATTR_PAD))
2986		return -EMSGSIZE;
2987
2988	return 0;
2989}
2990
2991static int nla_put_secy(struct macsec_secy *secy, struct sk_buff *skb)
2992{
2993	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
2994	struct nlattr *secy_nest = nla_nest_start_noflag(skb,
2995							 MACSEC_ATTR_SECY);
2996	u64 csid;
2997
2998	if (!secy_nest)
2999		return 1;
3000
3001	switch (secy->key_len) {
3002	case MACSEC_GCM_AES_128_SAK_LEN:
3003		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
3004		break;
3005	case MACSEC_GCM_AES_256_SAK_LEN:
3006		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
3007		break;
3008	default:
3009		goto cancel;
3010	}
3011
3012	if (nla_put_sci(skb, MACSEC_SECY_ATTR_SCI, secy->sci,
3013			MACSEC_SECY_ATTR_PAD) ||
3014	    nla_put_u64_64bit(skb, MACSEC_SECY_ATTR_CIPHER_SUITE,
3015			      csid, MACSEC_SECY_ATTR_PAD) ||
3016	    nla_put_u8(skb, MACSEC_SECY_ATTR_ICV_LEN, secy->icv_len) ||
3017	    nla_put_u8(skb, MACSEC_SECY_ATTR_OPER, secy->operational) ||
3018	    nla_put_u8(skb, MACSEC_SECY_ATTR_PROTECT, secy->protect_frames) ||
3019	    nla_put_u8(skb, MACSEC_SECY_ATTR_REPLAY, secy->replay_protect) ||
3020	    nla_put_u8(skb, MACSEC_SECY_ATTR_VALIDATE, secy->validate_frames) ||
3021	    nla_put_u8(skb, MACSEC_SECY_ATTR_ENCRYPT, tx_sc->encrypt) ||
3022	    nla_put_u8(skb, MACSEC_SECY_ATTR_INC_SCI, tx_sc->send_sci) ||
3023	    nla_put_u8(skb, MACSEC_SECY_ATTR_ES, tx_sc->end_station) ||
3024	    nla_put_u8(skb, MACSEC_SECY_ATTR_SCB, tx_sc->scb) ||
3025	    nla_put_u8(skb, MACSEC_SECY_ATTR_ENCODING_SA, tx_sc->encoding_sa))
3026		goto cancel;
3027
3028	if (secy->replay_protect) {
3029		if (nla_put_u32(skb, MACSEC_SECY_ATTR_WINDOW, secy->replay_window))
3030			goto cancel;
3031	}
3032
3033	nla_nest_end(skb, secy_nest);
3034	return 0;
3035
3036cancel:
3037	nla_nest_cancel(skb, secy_nest);
3038	return 1;
3039}
3040
3041static noinline_for_stack int
3042dump_secy(struct macsec_secy *secy, struct net_device *dev,
3043	  struct sk_buff *skb, struct netlink_callback *cb)
3044{
3045	struct macsec_tx_sc_stats tx_sc_stats = {0, };
3046	struct macsec_tx_sa_stats tx_sa_stats = {0, };
3047	struct macsec_rx_sc_stats rx_sc_stats = {0, };
3048	struct macsec_rx_sa_stats rx_sa_stats = {0, };
3049	struct macsec_dev *macsec = netdev_priv(dev);
3050	struct macsec_dev_stats dev_stats = {0, };
3051	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
3052	struct nlattr *txsa_list, *rxsc_list;
3053	struct macsec_rx_sc *rx_sc;
3054	struct nlattr *attr;
3055	void *hdr;
3056	int i, j;
3057
3058	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
3059			  &macsec_fam, NLM_F_MULTI, MACSEC_CMD_GET_TXSC);
3060	if (!hdr)
3061		return -EMSGSIZE;
3062
3063	genl_dump_check_consistent(cb, hdr);
3064
3065	if (nla_put_u32(skb, MACSEC_ATTR_IFINDEX, dev->ifindex))
3066		goto nla_put_failure;
3067
3068	attr = nla_nest_start_noflag(skb, MACSEC_ATTR_OFFLOAD);
3069	if (!attr)
3070		goto nla_put_failure;
3071	if (nla_put_u8(skb, MACSEC_OFFLOAD_ATTR_TYPE, macsec->offload))
3072		goto nla_put_failure;
3073	nla_nest_end(skb, attr);
3074
3075	if (nla_put_secy(secy, skb))
3076		goto nla_put_failure;
3077
3078	attr = nla_nest_start_noflag(skb, MACSEC_ATTR_TXSC_STATS);
3079	if (!attr)
3080		goto nla_put_failure;
3081
3082	get_tx_sc_stats(dev, &tx_sc_stats);
3083	if (copy_tx_sc_stats(skb, &tx_sc_stats)) {
3084		nla_nest_cancel(skb, attr);
3085		goto nla_put_failure;
3086	}
3087	nla_nest_end(skb, attr);
3088
3089	attr = nla_nest_start_noflag(skb, MACSEC_ATTR_SECY_STATS);
3090	if (!attr)
3091		goto nla_put_failure;
3092	get_secy_stats(dev, &dev_stats);
3093	if (copy_secy_stats(skb, &dev_stats)) {
3094		nla_nest_cancel(skb, attr);
3095		goto nla_put_failure;
3096	}
3097	nla_nest_end(skb, attr);
3098
3099	txsa_list = nla_nest_start_noflag(skb, MACSEC_ATTR_TXSA_LIST);
3100	if (!txsa_list)
3101		goto nla_put_failure;
3102	for (i = 0, j = 1; i < MACSEC_NUM_AN; i++) {
3103		struct macsec_tx_sa *tx_sa = rtnl_dereference(tx_sc->sa[i]);
3104		struct nlattr *txsa_nest;
3105		u64 pn;
3106		int pn_len;
3107
3108		if (!tx_sa)
3109			continue;
3110
3111		txsa_nest = nla_nest_start_noflag(skb, j++);
3112		if (!txsa_nest) {
3113			nla_nest_cancel(skb, txsa_list);
3114			goto nla_put_failure;
3115		}
3116
3117		attr = nla_nest_start_noflag(skb, MACSEC_SA_ATTR_STATS);
3118		if (!attr) {
3119			nla_nest_cancel(skb, txsa_nest);
3120			nla_nest_cancel(skb, txsa_list);
3121			goto nla_put_failure;
3122		}
3123		memset(&tx_sa_stats, 0, sizeof(tx_sa_stats));
3124		get_tx_sa_stats(dev, i, tx_sa, &tx_sa_stats);
3125		if (copy_tx_sa_stats(skb, &tx_sa_stats)) {
3126			nla_nest_cancel(skb, attr);
3127			nla_nest_cancel(skb, txsa_nest);
3128			nla_nest_cancel(skb, txsa_list);
3129			goto nla_put_failure;
3130		}
3131		nla_nest_end(skb, attr);
3132
3133		if (secy->xpn) {
3134			pn = tx_sa->next_pn;
3135			pn_len = MACSEC_XPN_PN_LEN;
3136		} else {
3137			pn = tx_sa->next_pn_halves.lower;
3138			pn_len = MACSEC_DEFAULT_PN_LEN;
3139		}
3140
3141		if (nla_put_u8(skb, MACSEC_SA_ATTR_AN, i) ||
3142		    nla_put(skb, MACSEC_SA_ATTR_PN, pn_len, &pn) ||
3143		    nla_put(skb, MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, tx_sa->key.id) ||
3144		    (secy->xpn && nla_put_ssci(skb, MACSEC_SA_ATTR_SSCI, tx_sa->ssci)) ||
3145		    nla_put_u8(skb, MACSEC_SA_ATTR_ACTIVE, tx_sa->active)) {
3146			nla_nest_cancel(skb, txsa_nest);
3147			nla_nest_cancel(skb, txsa_list);
3148			goto nla_put_failure;
3149		}
3150
3151		nla_nest_end(skb, txsa_nest);
3152	}
3153	nla_nest_end(skb, txsa_list);
3154
3155	rxsc_list = nla_nest_start_noflag(skb, MACSEC_ATTR_RXSC_LIST);
3156	if (!rxsc_list)
3157		goto nla_put_failure;
3158
3159	j = 1;
3160	for_each_rxsc_rtnl(secy, rx_sc) {
3161		int k;
3162		struct nlattr *rxsa_list;
3163		struct nlattr *rxsc_nest = nla_nest_start_noflag(skb, j++);
3164
3165		if (!rxsc_nest) {
3166			nla_nest_cancel(skb, rxsc_list);
3167			goto nla_put_failure;
3168		}
3169
3170		if (nla_put_u8(skb, MACSEC_RXSC_ATTR_ACTIVE, rx_sc->active) ||
3171		    nla_put_sci(skb, MACSEC_RXSC_ATTR_SCI, rx_sc->sci,
3172				MACSEC_RXSC_ATTR_PAD)) {
3173			nla_nest_cancel(skb, rxsc_nest);
3174			nla_nest_cancel(skb, rxsc_list);
3175			goto nla_put_failure;
3176		}
3177
3178		attr = nla_nest_start_noflag(skb, MACSEC_RXSC_ATTR_STATS);
3179		if (!attr) {
3180			nla_nest_cancel(skb, rxsc_nest);
3181			nla_nest_cancel(skb, rxsc_list);
3182			goto nla_put_failure;
3183		}
3184		memset(&rx_sc_stats, 0, sizeof(rx_sc_stats));
3185		get_rx_sc_stats(dev, rx_sc, &rx_sc_stats);
3186		if (copy_rx_sc_stats(skb, &rx_sc_stats)) {
3187			nla_nest_cancel(skb, attr);
3188			nla_nest_cancel(skb, rxsc_nest);
3189			nla_nest_cancel(skb, rxsc_list);
3190			goto nla_put_failure;
3191		}
3192		nla_nest_end(skb, attr);
3193
3194		rxsa_list = nla_nest_start_noflag(skb,
3195						  MACSEC_RXSC_ATTR_SA_LIST);
3196		if (!rxsa_list) {
3197			nla_nest_cancel(skb, rxsc_nest);
3198			nla_nest_cancel(skb, rxsc_list);
3199			goto nla_put_failure;
3200		}
3201
3202		for (i = 0, k = 1; i < MACSEC_NUM_AN; i++) {
3203			struct macsec_rx_sa *rx_sa = rtnl_dereference(rx_sc->sa[i]);
3204			struct nlattr *rxsa_nest;
3205			u64 pn;
3206			int pn_len;
3207
3208			if (!rx_sa)
3209				continue;
3210
3211			rxsa_nest = nla_nest_start_noflag(skb, k++);
3212			if (!rxsa_nest) {
3213				nla_nest_cancel(skb, rxsa_list);
3214				nla_nest_cancel(skb, rxsc_nest);
3215				nla_nest_cancel(skb, rxsc_list);
3216				goto nla_put_failure;
3217			}
3218
3219			attr = nla_nest_start_noflag(skb,
3220						     MACSEC_SA_ATTR_STATS);
3221			if (!attr) {
3222				nla_nest_cancel(skb, rxsa_list);
3223				nla_nest_cancel(skb, rxsc_nest);
3224				nla_nest_cancel(skb, rxsc_list);
3225				goto nla_put_failure;
3226			}
3227			memset(&rx_sa_stats, 0, sizeof(rx_sa_stats));
3228			get_rx_sa_stats(dev, rx_sc, i, rx_sa, &rx_sa_stats);
3229			if (copy_rx_sa_stats(skb, &rx_sa_stats)) {
3230				nla_nest_cancel(skb, attr);
3231				nla_nest_cancel(skb, rxsa_list);
3232				nla_nest_cancel(skb, rxsc_nest);
3233				nla_nest_cancel(skb, rxsc_list);
3234				goto nla_put_failure;
3235			}
3236			nla_nest_end(skb, attr);
3237
3238			if (secy->xpn) {
3239				pn = rx_sa->next_pn;
3240				pn_len = MACSEC_XPN_PN_LEN;
3241			} else {
3242				pn = rx_sa->next_pn_halves.lower;
3243				pn_len = MACSEC_DEFAULT_PN_LEN;
3244			}
3245
3246			if (nla_put_u8(skb, MACSEC_SA_ATTR_AN, i) ||
3247			    nla_put(skb, MACSEC_SA_ATTR_PN, pn_len, &pn) ||
3248			    nla_put(skb, MACSEC_SA_ATTR_KEYID, MACSEC_KEYID_LEN, rx_sa->key.id) ||
3249			    (secy->xpn && nla_put_ssci(skb, MACSEC_SA_ATTR_SSCI, rx_sa->ssci)) ||
3250			    nla_put_u8(skb, MACSEC_SA_ATTR_ACTIVE, rx_sa->active)) {
3251				nla_nest_cancel(skb, rxsa_nest);
3252				nla_nest_cancel(skb, rxsc_nest);
3253				nla_nest_cancel(skb, rxsc_list);
3254				goto nla_put_failure;
3255			}
3256			nla_nest_end(skb, rxsa_nest);
3257		}
3258
3259		nla_nest_end(skb, rxsa_list);
3260		nla_nest_end(skb, rxsc_nest);
3261	}
3262
3263	nla_nest_end(skb, rxsc_list);
3264
3265	genlmsg_end(skb, hdr);
3266
3267	return 0;
3268
3269nla_put_failure:
3270	genlmsg_cancel(skb, hdr);
3271	return -EMSGSIZE;
3272}
3273
3274static int macsec_generation = 1; /* protected by RTNL */
3275
3276static int macsec_dump_txsc(struct sk_buff *skb, struct netlink_callback *cb)
3277{
3278	struct net *net = sock_net(skb->sk);
3279	struct net_device *dev;
3280	int dev_idx, d;
3281
3282	dev_idx = cb->args[0];
3283
3284	d = 0;
3285	rtnl_lock();
3286
3287	cb->seq = macsec_generation;
3288
3289	for_each_netdev(net, dev) {
3290		struct macsec_secy *secy;
3291
3292		if (d < dev_idx)
3293			goto next;
3294
3295		if (!netif_is_macsec(dev))
3296			goto next;
3297
3298		secy = &macsec_priv(dev)->secy;
3299		if (dump_secy(secy, dev, skb, cb) < 0)
3300			goto done;
3301next:
3302		d++;
3303	}
3304
3305done:
3306	rtnl_unlock();
3307	cb->args[0] = d;
3308	return skb->len;
3309}
3310
3311static const struct genl_small_ops macsec_genl_ops[] = {
3312	{
3313		.cmd = MACSEC_CMD_GET_TXSC,
3314		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3315		.dumpit = macsec_dump_txsc,
3316	},
3317	{
3318		.cmd = MACSEC_CMD_ADD_RXSC,
3319		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3320		.doit = macsec_add_rxsc,
3321		.flags = GENL_ADMIN_PERM,
3322	},
3323	{
3324		.cmd = MACSEC_CMD_DEL_RXSC,
3325		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3326		.doit = macsec_del_rxsc,
3327		.flags = GENL_ADMIN_PERM,
3328	},
3329	{
3330		.cmd = MACSEC_CMD_UPD_RXSC,
3331		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3332		.doit = macsec_upd_rxsc,
3333		.flags = GENL_ADMIN_PERM,
3334	},
3335	{
3336		.cmd = MACSEC_CMD_ADD_TXSA,
3337		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3338		.doit = macsec_add_txsa,
3339		.flags = GENL_ADMIN_PERM,
3340	},
3341	{
3342		.cmd = MACSEC_CMD_DEL_TXSA,
3343		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3344		.doit = macsec_del_txsa,
3345		.flags = GENL_ADMIN_PERM,
3346	},
3347	{
3348		.cmd = MACSEC_CMD_UPD_TXSA,
3349		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3350		.doit = macsec_upd_txsa,
3351		.flags = GENL_ADMIN_PERM,
3352	},
3353	{
3354		.cmd = MACSEC_CMD_ADD_RXSA,
3355		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3356		.doit = macsec_add_rxsa,
3357		.flags = GENL_ADMIN_PERM,
3358	},
3359	{
3360		.cmd = MACSEC_CMD_DEL_RXSA,
3361		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3362		.doit = macsec_del_rxsa,
3363		.flags = GENL_ADMIN_PERM,
3364	},
3365	{
3366		.cmd = MACSEC_CMD_UPD_RXSA,
3367		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3368		.doit = macsec_upd_rxsa,
3369		.flags = GENL_ADMIN_PERM,
3370	},
3371	{
3372		.cmd = MACSEC_CMD_UPD_OFFLOAD,
3373		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
3374		.doit = macsec_upd_offload,
3375		.flags = GENL_ADMIN_PERM,
3376	},
3377};
3378
3379static struct genl_family macsec_fam __ro_after_init = {
3380	.name		= MACSEC_GENL_NAME,
3381	.hdrsize	= 0,
3382	.version	= MACSEC_GENL_VERSION,
3383	.maxattr	= MACSEC_ATTR_MAX,
3384	.policy = macsec_genl_policy,
3385	.netnsok	= true,
3386	.module		= THIS_MODULE,
3387	.small_ops	= macsec_genl_ops,
3388	.n_small_ops	= ARRAY_SIZE(macsec_genl_ops),
3389	.resv_start_op	= MACSEC_CMD_UPD_OFFLOAD + 1,
3390};
3391
3392static struct sk_buff *macsec_insert_tx_tag(struct sk_buff *skb,
3393					    struct net_device *dev)
3394{
3395	struct macsec_dev *macsec = macsec_priv(dev);
3396	const struct macsec_ops *ops;
3397	struct phy_device *phydev;
3398	struct macsec_context ctx;
3399	int skb_final_len;
3400	int err;
3401
3402	ops = macsec_get_ops(macsec, &ctx);
3403	skb_final_len = skb->len - ETH_HLEN + ops->needed_headroom +
3404		ops->needed_tailroom;
3405	if (unlikely(skb_final_len > macsec->real_dev->mtu)) {
3406		err = -EINVAL;
3407		goto cleanup;
3408	}
3409
3410	phydev = macsec->real_dev->phydev;
3411
3412	err = skb_ensure_writable_head_tail(skb, dev);
3413	if (unlikely(err < 0))
3414		goto cleanup;
3415
3416	err = ops->mdo_insert_tx_tag(phydev, skb);
3417	if (unlikely(err))
3418		goto cleanup;
3419
3420	return skb;
3421cleanup:
3422	kfree_skb(skb);
3423	return ERR_PTR(err);
3424}
3425
3426static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
3427				     struct net_device *dev)
3428{
3429	struct macsec_dev *macsec = netdev_priv(dev);
3430	struct macsec_secy *secy = &macsec->secy;
3431	struct pcpu_secy_stats *secy_stats;
3432	int ret, len;
3433
3434	if (macsec_is_offloaded(netdev_priv(dev))) {
3435		struct metadata_dst *md_dst = secy->tx_sc.md_dst;
3436
3437		skb_dst_drop(skb);
3438		dst_hold(&md_dst->dst);
3439		skb_dst_set(skb, &md_dst->dst);
3440
3441		if (macsec->insert_tx_tag) {
3442			skb = macsec_insert_tx_tag(skb, dev);
3443			if (IS_ERR(skb)) {
3444				DEV_STATS_INC(dev, tx_dropped);
3445				return NETDEV_TX_OK;
3446			}
3447		}
3448
3449		skb->dev = macsec->real_dev;
3450		return dev_queue_xmit(skb);
3451	}
3452
3453	/* 10.5 */
3454	if (!secy->protect_frames) {
3455		secy_stats = this_cpu_ptr(macsec->stats);
3456		u64_stats_update_begin(&secy_stats->syncp);
3457		secy_stats->stats.OutPktsUntagged++;
3458		u64_stats_update_end(&secy_stats->syncp);
3459		skb->dev = macsec->real_dev;
3460		len = skb->len;
3461		ret = dev_queue_xmit(skb);
3462		count_tx(dev, ret, len);
3463		return ret;
3464	}
3465
3466	if (!secy->operational) {
3467		kfree_skb(skb);
3468		DEV_STATS_INC(dev, tx_dropped);
3469		return NETDEV_TX_OK;
3470	}
3471
3472	len = skb->len;
3473	skb = macsec_encrypt(skb, dev);
3474	if (IS_ERR(skb)) {
3475		if (PTR_ERR(skb) != -EINPROGRESS)
3476			DEV_STATS_INC(dev, tx_dropped);
3477		return NETDEV_TX_OK;
3478	}
3479
3480	macsec_count_tx(skb, &macsec->secy.tx_sc, macsec_skb_cb(skb)->tx_sa);
3481
3482	macsec_encrypt_finish(skb, dev);
3483	ret = dev_queue_xmit(skb);
3484	count_tx(dev, ret, len);
3485	return ret;
3486}
3487
3488#define MACSEC_FEATURES \
3489	(NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST)
3490
3491#define MACSEC_OFFLOAD_FEATURES \
3492	(MACSEC_FEATURES | NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES | \
3493	 NETIF_F_LRO | NETIF_F_RXHASH | NETIF_F_CSUM_MASK | NETIF_F_RXCSUM)
3494
3495static int macsec_dev_init(struct net_device *dev)
3496{
3497	struct macsec_dev *macsec = macsec_priv(dev);
3498	struct net_device *real_dev = macsec->real_dev;
3499	int err;
3500
3501	err = gro_cells_init(&macsec->gro_cells, dev);
3502	if (err)
3503		return err;
3504
3505	macsec_inherit_tso_max(dev);
3506
3507	dev->hw_features = real_dev->hw_features & MACSEC_OFFLOAD_FEATURES;
3508	dev->hw_features |= NETIF_F_GSO_SOFTWARE;
3509
3510	dev->features = real_dev->features & MACSEC_OFFLOAD_FEATURES;
3511	dev->features |= NETIF_F_GSO_SOFTWARE;
3512	dev->lltx = true;
3513	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
3514
3515	macsec_set_head_tail_room(dev);
3516
3517	if (is_zero_ether_addr(dev->dev_addr))
3518		eth_hw_addr_inherit(dev, real_dev);
3519	if (is_zero_ether_addr(dev->broadcast))
3520		memcpy(dev->broadcast, real_dev->broadcast, dev->addr_len);
3521
3522	/* Get macsec's reference to real_dev */
3523	netdev_hold(real_dev, &macsec->dev_tracker, GFP_KERNEL);
3524
3525	return 0;
3526}
3527
3528static void macsec_dev_uninit(struct net_device *dev)
3529{
3530	struct macsec_dev *macsec = macsec_priv(dev);
3531
3532	gro_cells_destroy(&macsec->gro_cells);
3533}
3534
3535static netdev_features_t macsec_fix_features(struct net_device *dev,
3536					     netdev_features_t features)
3537{
3538	struct macsec_dev *macsec = macsec_priv(dev);
3539	struct net_device *real_dev = macsec->real_dev;
3540	netdev_features_t mask;
3541
3542	mask = macsec_is_offloaded(macsec) ? MACSEC_OFFLOAD_FEATURES
3543					   : MACSEC_FEATURES;
3544
3545	features &= (real_dev->features & mask) |
3546		    NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES;
3547
3548	return features;
3549}
3550
3551static int macsec_dev_open(struct net_device *dev)
3552{
3553	struct macsec_dev *macsec = macsec_priv(dev);
3554	struct net_device *real_dev = macsec->real_dev;
3555	int err;
3556
3557	err = dev_uc_add(real_dev, dev->dev_addr);
3558	if (err < 0)
3559		return err;
3560
3561	if (dev->flags & IFF_ALLMULTI) {
3562		err = dev_set_allmulti(real_dev, 1);
3563		if (err < 0)
3564			goto del_unicast;
3565	}
3566
3567	if (dev->flags & IFF_PROMISC) {
3568		err = dev_set_promiscuity(real_dev, 1);
3569		if (err < 0)
3570			goto clear_allmulti;
3571	}
3572
3573	/* If h/w offloading is available, propagate to the device */
3574	if (macsec_is_offloaded(macsec)) {
3575		const struct macsec_ops *ops;
3576		struct macsec_context ctx;
3577
3578		ops = macsec_get_ops(netdev_priv(dev), &ctx);
3579		if (!ops) {
3580			err = -EOPNOTSUPP;
3581			goto clear_allmulti;
3582		}
3583
3584		ctx.secy = &macsec->secy;
3585		err = macsec_offload(ops->mdo_dev_open, &ctx);
3586		if (err)
3587			goto clear_allmulti;
3588	}
3589
3590	if (netif_carrier_ok(real_dev))
3591		netif_carrier_on(dev);
3592
3593	return 0;
3594clear_allmulti:
3595	if (dev->flags & IFF_ALLMULTI)
3596		dev_set_allmulti(real_dev, -1);
3597del_unicast:
3598	dev_uc_del(real_dev, dev->dev_addr);
3599	netif_carrier_off(dev);
3600	return err;
3601}
3602
3603static int macsec_dev_stop(struct net_device *dev)
3604{
3605	struct macsec_dev *macsec = macsec_priv(dev);
3606	struct net_device *real_dev = macsec->real_dev;
3607
3608	netif_carrier_off(dev);
3609
3610	/* If h/w offloading is available, propagate to the device */
3611	if (macsec_is_offloaded(macsec)) {
3612		const struct macsec_ops *ops;
3613		struct macsec_context ctx;
3614
3615		ops = macsec_get_ops(macsec, &ctx);
3616		if (ops) {
3617			ctx.secy = &macsec->secy;
3618			macsec_offload(ops->mdo_dev_stop, &ctx);
3619		}
3620	}
3621
3622	dev_mc_unsync(real_dev, dev);
3623	dev_uc_unsync(real_dev, dev);
3624
3625	if (dev->flags & IFF_ALLMULTI)
3626		dev_set_allmulti(real_dev, -1);
3627
3628	if (dev->flags & IFF_PROMISC)
3629		dev_set_promiscuity(real_dev, -1);
3630
3631	dev_uc_del(real_dev, dev->dev_addr);
3632
3633	return 0;
3634}
3635
3636static void macsec_dev_change_rx_flags(struct net_device *dev, int change)
3637{
3638	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3639
3640	if (!(dev->flags & IFF_UP))
3641		return;
3642
3643	if (change & IFF_ALLMULTI)
3644		dev_set_allmulti(real_dev, dev->flags & IFF_ALLMULTI ? 1 : -1);
3645
3646	if (change & IFF_PROMISC)
3647		dev_set_promiscuity(real_dev,
3648				    dev->flags & IFF_PROMISC ? 1 : -1);
3649}
3650
3651static void macsec_dev_set_rx_mode(struct net_device *dev)
3652{
3653	struct net_device *real_dev = macsec_priv(dev)->real_dev;
3654
3655	dev_mc_sync(real_dev, dev);
3656	dev_uc_sync(real_dev, dev);
3657}
3658
3659static int macsec_set_mac_address(struct net_device *dev, void *p)
3660{
3661	struct macsec_dev *macsec = macsec_priv(dev);
3662	struct net_device *real_dev = macsec->real_dev;
3663	struct sockaddr *addr = p;
3664	u8  old_addr[ETH_ALEN];
3665	int err;
3666
3667	if (!is_valid_ether_addr(addr->sa_data))
3668		return -EADDRNOTAVAIL;
3669
3670	if (dev->flags & IFF_UP) {
3671		err = dev_uc_add(real_dev, addr->sa_data);
3672		if (err < 0)
3673			return err;
3674	}
3675
3676	ether_addr_copy(old_addr, dev->dev_addr);
3677	eth_hw_addr_set(dev, addr->sa_data);
3678
3679	/* If h/w offloading is available, propagate to the device */
3680	if (macsec_is_offloaded(macsec)) {
3681		const struct macsec_ops *ops;
3682		struct macsec_context ctx;
3683
3684		ops = macsec_get_ops(macsec, &ctx);
3685		if (!ops) {
3686			err = -EOPNOTSUPP;
3687			goto restore_old_addr;
3688		}
3689
3690		ctx.secy = &macsec->secy;
3691		err = macsec_offload(ops->mdo_upd_secy, &ctx);
3692		if (err)
3693			goto restore_old_addr;
3694	}
3695
3696	if (dev->flags & IFF_UP)
3697		dev_uc_del(real_dev, old_addr);
3698
3699	return 0;
3700
3701restore_old_addr:
3702	if (dev->flags & IFF_UP)
3703		dev_uc_del(real_dev, addr->sa_data);
3704
3705	eth_hw_addr_set(dev, old_addr);
3706
3707	return err;
3708}
3709
3710static int macsec_change_mtu(struct net_device *dev, int new_mtu)
3711{
3712	struct macsec_dev *macsec = macsec_priv(dev);
3713	unsigned int extra = macsec->secy.icv_len + macsec_extra_len(true);
3714
3715	if (macsec->real_dev->mtu - extra < new_mtu)
3716		return -ERANGE;
3717
3718	WRITE_ONCE(dev->mtu, new_mtu);
3719
3720	return 0;
3721}
3722
3723static void macsec_get_stats64(struct net_device *dev,
3724			       struct rtnl_link_stats64 *s)
3725{
3726	if (!dev->tstats)
3727		return;
3728
3729	dev_fetch_sw_netstats(s, dev->tstats);
3730
3731	s->rx_dropped = DEV_STATS_READ(dev, rx_dropped);
3732	s->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
3733	s->rx_errors = DEV_STATS_READ(dev, rx_errors);
3734}
3735
3736static int macsec_get_iflink(const struct net_device *dev)
3737{
3738	return READ_ONCE(macsec_priv(dev)->real_dev->ifindex);
3739}
3740
3741static const struct net_device_ops macsec_netdev_ops = {
3742	.ndo_init		= macsec_dev_init,
3743	.ndo_uninit		= macsec_dev_uninit,
3744	.ndo_open		= macsec_dev_open,
3745	.ndo_stop		= macsec_dev_stop,
3746	.ndo_fix_features	= macsec_fix_features,
3747	.ndo_change_mtu		= macsec_change_mtu,
3748	.ndo_set_rx_mode	= macsec_dev_set_rx_mode,
3749	.ndo_change_rx_flags	= macsec_dev_change_rx_flags,
3750	.ndo_set_mac_address	= macsec_set_mac_address,
3751	.ndo_start_xmit		= macsec_start_xmit,
3752	.ndo_get_stats64	= macsec_get_stats64,
3753	.ndo_get_iflink		= macsec_get_iflink,
3754};
3755
3756static const struct device_type macsec_type = {
3757	.name = "macsec",
3758};
3759
3760static int validate_cipher_suite(const struct nlattr *attr,
3761				 struct netlink_ext_ack *extack);
3762static const struct nla_policy macsec_rtnl_policy[IFLA_MACSEC_MAX + 1] = {
3763	[IFLA_MACSEC_SCI] = { .type = NLA_U64 },
3764	[IFLA_MACSEC_PORT] = { .type = NLA_U16 },
3765	[IFLA_MACSEC_ICV_LEN] = NLA_POLICY_RANGE(NLA_U8, MACSEC_MIN_ICV_LEN, MACSEC_STD_ICV_LEN),
3766	[IFLA_MACSEC_CIPHER_SUITE] = NLA_POLICY_VALIDATE_FN(NLA_U64, validate_cipher_suite),
3767	[IFLA_MACSEC_WINDOW] = { .type = NLA_U32 },
3768	[IFLA_MACSEC_ENCODING_SA] = NLA_POLICY_MAX(NLA_U8, MACSEC_NUM_AN - 1),
3769	[IFLA_MACSEC_ENCRYPT] = NLA_POLICY_MAX(NLA_U8, 1),
3770	[IFLA_MACSEC_PROTECT] = NLA_POLICY_MAX(NLA_U8, 1),
3771	[IFLA_MACSEC_INC_SCI] = NLA_POLICY_MAX(NLA_U8, 1),
3772	[IFLA_MACSEC_ES] = NLA_POLICY_MAX(NLA_U8, 1),
3773	[IFLA_MACSEC_SCB] = NLA_POLICY_MAX(NLA_U8, 1),
3774	[IFLA_MACSEC_REPLAY_PROTECT] = NLA_POLICY_MAX(NLA_U8, 1),
3775	[IFLA_MACSEC_VALIDATION] = NLA_POLICY_MAX(NLA_U8, MACSEC_VALIDATE_MAX),
3776	[IFLA_MACSEC_OFFLOAD] = NLA_POLICY_MAX(NLA_U8, MACSEC_OFFLOAD_MAX),
3777};
3778
3779static void macsec_free_netdev(struct net_device *dev)
3780{
3781	struct macsec_dev *macsec = macsec_priv(dev);
3782
3783	dst_release(&macsec->secy.tx_sc.md_dst->dst);
3784	free_percpu(macsec->stats);
3785	free_percpu(macsec->secy.tx_sc.stats);
3786
3787	/* Get rid of the macsec's reference to real_dev */
3788	netdev_put(macsec->real_dev, &macsec->dev_tracker);
3789}
3790
3791static void macsec_setup(struct net_device *dev)
3792{
3793	ether_setup(dev);
3794	dev->min_mtu = 0;
3795	dev->max_mtu = ETH_MAX_MTU;
3796	dev->priv_flags |= IFF_NO_QUEUE | IFF_UNICAST_FLT;
3797	dev->netdev_ops = &macsec_netdev_ops;
3798	dev->needs_free_netdev = true;
3799	dev->priv_destructor = macsec_free_netdev;
3800	SET_NETDEV_DEVTYPE(dev, &macsec_type);
3801
3802	eth_zero_addr(dev->broadcast);
3803}
3804
3805static int macsec_changelink_common(struct net_device *dev,
3806				    struct nlattr *data[])
3807{
3808	struct macsec_secy *secy;
3809	struct macsec_tx_sc *tx_sc;
3810
3811	secy = &macsec_priv(dev)->secy;
3812	tx_sc = &secy->tx_sc;
3813
3814	if (data[IFLA_MACSEC_ENCODING_SA]) {
3815		struct macsec_tx_sa *tx_sa;
3816
3817		tx_sc->encoding_sa = nla_get_u8(data[IFLA_MACSEC_ENCODING_SA]);
3818		tx_sa = rtnl_dereference(tx_sc->sa[tx_sc->encoding_sa]);
3819
3820		secy->operational = tx_sa && tx_sa->active;
3821	}
3822
3823	if (data[IFLA_MACSEC_ENCRYPT])
3824		tx_sc->encrypt = !!nla_get_u8(data[IFLA_MACSEC_ENCRYPT]);
3825
3826	if (data[IFLA_MACSEC_PROTECT])
3827		secy->protect_frames = !!nla_get_u8(data[IFLA_MACSEC_PROTECT]);
3828
3829	if (data[IFLA_MACSEC_INC_SCI])
3830		tx_sc->send_sci = !!nla_get_u8(data[IFLA_MACSEC_INC_SCI]);
3831
3832	if (data[IFLA_MACSEC_ES])
3833		tx_sc->end_station = !!nla_get_u8(data[IFLA_MACSEC_ES]);
3834
3835	if (data[IFLA_MACSEC_SCB])
3836		tx_sc->scb = !!nla_get_u8(data[IFLA_MACSEC_SCB]);
3837
3838	if (data[IFLA_MACSEC_REPLAY_PROTECT])
3839		secy->replay_protect = !!nla_get_u8(data[IFLA_MACSEC_REPLAY_PROTECT]);
3840
3841	if (data[IFLA_MACSEC_VALIDATION])
3842		secy->validate_frames = nla_get_u8(data[IFLA_MACSEC_VALIDATION]);
3843
3844	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
3845		switch (nla_get_u64(data[IFLA_MACSEC_CIPHER_SUITE])) {
3846		case MACSEC_CIPHER_ID_GCM_AES_128:
3847		case MACSEC_DEFAULT_CIPHER_ID:
3848			secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3849			secy->xpn = false;
3850			break;
3851		case MACSEC_CIPHER_ID_GCM_AES_256:
3852			secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3853			secy->xpn = false;
3854			break;
3855		case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
3856			secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
3857			secy->xpn = true;
3858			break;
3859		case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
3860			secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
3861			secy->xpn = true;
3862			break;
3863		default:
3864			return -EINVAL;
3865		}
3866	}
3867
3868	if (data[IFLA_MACSEC_WINDOW]) {
3869		secy->replay_window = nla_get_u32(data[IFLA_MACSEC_WINDOW]);
3870
3871		/* IEEE 802.1AEbw-2013 10.7.8 - maximum replay window
3872		 * for XPN cipher suites */
3873		if (secy->xpn &&
3874		    secy->replay_window > MACSEC_XPN_MAX_REPLAY_WINDOW)
3875			return -EINVAL;
3876	}
3877
3878	return 0;
3879}
3880
3881static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
3882			     struct nlattr *data[],
3883			     struct netlink_ext_ack *extack)
3884{
3885	struct macsec_dev *macsec = macsec_priv(dev);
3886	bool macsec_offload_state_change = false;
3887	enum macsec_offload offload;
3888	struct macsec_tx_sc tx_sc;
3889	struct macsec_secy secy;
3890	int ret;
3891
3892	if (!data)
3893		return 0;
3894
3895	if (data[IFLA_MACSEC_CIPHER_SUITE] ||
3896	    data[IFLA_MACSEC_ICV_LEN] ||
3897	    data[IFLA_MACSEC_SCI] ||
3898	    data[IFLA_MACSEC_PORT])
3899		return -EINVAL;
3900
3901	/* Keep a copy of unmodified secy and tx_sc, in case the offload
3902	 * propagation fails, to revert macsec_changelink_common.
3903	 */
3904	memcpy(&secy, &macsec->secy, sizeof(secy));
3905	memcpy(&tx_sc, &macsec->secy.tx_sc, sizeof(tx_sc));
3906
3907	ret = macsec_changelink_common(dev, data);
3908	if (ret)
3909		goto cleanup;
3910
3911	if (data[IFLA_MACSEC_OFFLOAD]) {
3912		offload = nla_get_u8(data[IFLA_MACSEC_OFFLOAD]);
3913		if (macsec->offload != offload) {
3914			macsec_offload_state_change = true;
3915			ret = macsec_update_offload(dev, offload);
3916			if (ret)
3917				goto cleanup;
3918		}
3919	}
3920
3921	/* If h/w offloading is available, propagate to the device */
3922	if (!macsec_offload_state_change && macsec_is_offloaded(macsec)) {
3923		const struct macsec_ops *ops;
3924		struct macsec_context ctx;
3925
3926		ops = macsec_get_ops(netdev_priv(dev), &ctx);
3927		if (!ops) {
3928			ret = -EOPNOTSUPP;
3929			goto cleanup;
3930		}
3931
3932		ctx.secy = &macsec->secy;
3933		ret = macsec_offload(ops->mdo_upd_secy, &ctx);
3934		if (ret)
3935			goto cleanup;
3936	}
3937
3938	return 0;
3939
3940cleanup:
3941	memcpy(&macsec->secy.tx_sc, &tx_sc, sizeof(tx_sc));
3942	memcpy(&macsec->secy, &secy, sizeof(secy));
3943
3944	return ret;
3945}
3946
3947static void macsec_del_dev(struct macsec_dev *macsec)
3948{
3949	int i;
3950
3951	while (macsec->secy.rx_sc) {
3952		struct macsec_rx_sc *rx_sc = rtnl_dereference(macsec->secy.rx_sc);
3953
3954		rcu_assign_pointer(macsec->secy.rx_sc, rx_sc->next);
3955		free_rx_sc(rx_sc);
3956	}
3957
3958	for (i = 0; i < MACSEC_NUM_AN; i++) {
3959		struct macsec_tx_sa *sa = rtnl_dereference(macsec->secy.tx_sc.sa[i]);
3960
3961		if (sa) {
3962			RCU_INIT_POINTER(macsec->secy.tx_sc.sa[i], NULL);
3963			clear_tx_sa(sa);
3964		}
3965	}
3966}
3967
3968static void macsec_common_dellink(struct net_device *dev, struct list_head *head)
3969{
3970	struct macsec_dev *macsec = macsec_priv(dev);
3971	struct net_device *real_dev = macsec->real_dev;
3972
3973	/* If h/w offloading is available, propagate to the device */
3974	if (macsec_is_offloaded(macsec)) {
3975		const struct macsec_ops *ops;
3976		struct macsec_context ctx;
3977
3978		ops = macsec_get_ops(netdev_priv(dev), &ctx);
3979		if (ops) {
3980			ctx.secy = &macsec->secy;
3981			macsec_offload(ops->mdo_del_secy, &ctx);
3982		}
3983	}
3984
3985	unregister_netdevice_queue(dev, head);
3986	list_del_rcu(&macsec->secys);
3987	macsec_del_dev(macsec);
3988	netdev_upper_dev_unlink(real_dev, dev);
3989
3990	macsec_generation++;
3991}
3992
3993static void macsec_dellink(struct net_device *dev, struct list_head *head)
3994{
3995	struct macsec_dev *macsec = macsec_priv(dev);
3996	struct net_device *real_dev = macsec->real_dev;
3997	struct macsec_rxh_data *rxd = macsec_data_rtnl(real_dev);
3998
3999	macsec_common_dellink(dev, head);
4000
4001	if (list_empty(&rxd->secys)) {
4002		netdev_rx_handler_unregister(real_dev);
4003		kfree(rxd);
4004	}
4005}
4006
4007static int register_macsec_dev(struct net_device *real_dev,
4008			       struct net_device *dev)
4009{
4010	struct macsec_dev *macsec = macsec_priv(dev);
4011	struct macsec_rxh_data *rxd = macsec_data_rtnl(real_dev);
4012
4013	if (!rxd) {
4014		int err;
4015
4016		rxd = kmalloc(sizeof(*rxd), GFP_KERNEL);
4017		if (!rxd)
4018			return -ENOMEM;
4019
4020		INIT_LIST_HEAD(&rxd->secys);
4021
4022		err = netdev_rx_handler_register(real_dev, macsec_handle_frame,
4023						 rxd);
4024		if (err < 0) {
4025			kfree(rxd);
4026			return err;
4027		}
4028	}
4029
4030	list_add_tail_rcu(&macsec->secys, &rxd->secys);
4031	return 0;
4032}
4033
4034static bool sci_exists(struct net_device *dev, sci_t sci)
4035{
4036	struct macsec_rxh_data *rxd = macsec_data_rtnl(dev);
4037	struct macsec_dev *macsec;
4038
4039	list_for_each_entry(macsec, &rxd->secys, secys) {
4040		if (macsec->secy.sci == sci)
4041			return true;
4042	}
4043
4044	return false;
4045}
4046
4047static sci_t dev_to_sci(struct net_device *dev, __be16 port)
4048{
4049	return make_sci(dev->dev_addr, port);
4050}
4051
4052static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
4053{
4054	struct macsec_dev *macsec = macsec_priv(dev);
4055	struct macsec_secy *secy = &macsec->secy;
4056
4057	macsec->stats = netdev_alloc_pcpu_stats(struct pcpu_secy_stats);
4058	if (!macsec->stats)
4059		return -ENOMEM;
4060
4061	secy->tx_sc.stats = netdev_alloc_pcpu_stats(struct pcpu_tx_sc_stats);
4062	if (!secy->tx_sc.stats)
4063		return -ENOMEM;
4064
4065	secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL);
4066	if (!secy->tx_sc.md_dst)
4067		/* macsec and secy percpu stats will be freed when unregistering
4068		 * net_device in macsec_free_netdev()
4069		 */
4070		return -ENOMEM;
4071
4072	if (sci == MACSEC_UNDEF_SCI)
4073		sci = dev_to_sci(dev, MACSEC_PORT_ES);
4074
4075	secy->netdev = dev;
4076	secy->operational = true;
4077	secy->key_len = DEFAULT_SAK_LEN;
4078	secy->icv_len = icv_len;
4079	secy->validate_frames = MACSEC_VALIDATE_DEFAULT;
4080	secy->protect_frames = true;
4081	secy->replay_protect = false;
4082	secy->xpn = DEFAULT_XPN;
4083
4084	secy->sci = sci;
4085	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
4086	secy->tx_sc.active = true;
4087	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
4088	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
4089	secy->tx_sc.send_sci = DEFAULT_SEND_SCI;
4090	secy->tx_sc.end_station = false;
4091	secy->tx_sc.scb = false;
4092
4093	return 0;
4094}
4095
4096static struct lock_class_key macsec_netdev_addr_lock_key;
4097
4098static int macsec_newlink(struct net_device *dev,
4099			  struct rtnl_newlink_params *params,
4100			  struct netlink_ext_ack *extack)
4101{
4102	struct net *link_net = rtnl_newlink_link_net(params);
4103	struct macsec_dev *macsec = macsec_priv(dev);
4104	struct nlattr **data = params->data;
4105	struct nlattr **tb = params->tb;
4106	rx_handler_func_t *rx_handler;
4107	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4108	struct net_device *real_dev;
4109	int err, mtu;
4110	sci_t sci;
4111
4112	if (!tb[IFLA_LINK])
4113		return -EINVAL;
4114	real_dev = __dev_get_by_index(link_net, nla_get_u32(tb[IFLA_LINK]));
4115	if (!real_dev)
4116		return -ENODEV;
4117	if (real_dev->type != ARPHRD_ETHER)
4118		return -EINVAL;
4119
4120	dev->priv_flags |= IFF_MACSEC;
4121
4122	macsec->real_dev = real_dev;
4123
4124	if (data && data[IFLA_MACSEC_OFFLOAD])
4125		macsec->offload = nla_get_offload(data[IFLA_MACSEC_OFFLOAD]);
4126	else
4127		/* MACsec offloading is off by default */
4128		macsec->offload = MACSEC_OFFLOAD_OFF;
4129
4130	/* Check if the offloading mode is supported by the underlying layers */
4131	if (macsec->offload != MACSEC_OFFLOAD_OFF &&
4132	    !macsec_check_offload(macsec->offload, macsec))
4133		return -EOPNOTSUPP;
4134
4135	/* send_sci must be set to true when transmit sci explicitly is set */
4136	if ((data && data[IFLA_MACSEC_SCI]) &&
4137	    (data && data[IFLA_MACSEC_INC_SCI])) {
4138		u8 send_sci = !!nla_get_u8(data[IFLA_MACSEC_INC_SCI]);
4139
4140		if (!send_sci)
4141			return -EINVAL;
4142	}
4143
4144	if (data && data[IFLA_MACSEC_ICV_LEN])
4145		icv_len = nla_get_u8(data[IFLA_MACSEC_ICV_LEN]);
4146	mtu = real_dev->mtu - icv_len - macsec_extra_len(true);
4147	if (mtu < 0)
4148		dev->mtu = 0;
4149	else
4150		dev->mtu = mtu;
4151
4152	rx_handler = rtnl_dereference(real_dev->rx_handler);
4153	if (rx_handler && rx_handler != macsec_handle_frame)
4154		return -EBUSY;
4155
4156	err = register_netdevice(dev);
4157	if (err < 0)
4158		return err;
4159
4160	netdev_lockdep_set_classes(dev);
4161	lockdep_set_class(&dev->addr_list_lock,
4162			  &macsec_netdev_addr_lock_key);
4163
4164	err = netdev_upper_dev_link(real_dev, dev, extack);
4165	if (err < 0)
4166		goto unregister;
4167
4168	/* need to be already registered so that ->init has run and
4169	 * the MAC addr is set
4170	 */
4171	if (data && data[IFLA_MACSEC_SCI])
4172		sci = nla_get_sci(data[IFLA_MACSEC_SCI]);
4173	else if (data && data[IFLA_MACSEC_PORT])
4174		sci = dev_to_sci(dev, nla_get_be16(data[IFLA_MACSEC_PORT]));
4175	else
4176		sci = dev_to_sci(dev, MACSEC_PORT_ES);
4177
4178	if (rx_handler && sci_exists(real_dev, sci)) {
4179		err = -EBUSY;
4180		goto unlink;
4181	}
4182
4183	err = macsec_add_dev(dev, sci, icv_len);
4184	if (err)
4185		goto unlink;
4186
4187	if (data) {
4188		err = macsec_changelink_common(dev, data);
4189		if (err)
4190			goto del_dev;
4191	}
4192
4193	/* If h/w offloading is available, propagate to the device */
4194	if (macsec_is_offloaded(macsec)) {
4195		const struct macsec_ops *ops;
4196		struct macsec_context ctx;
4197
4198		ops = macsec_get_ops(macsec, &ctx);
4199		if (ops) {
4200			ctx.secy = &macsec->secy;
4201			err = macsec_offload(ops->mdo_add_secy, &ctx);
4202			if (err)
4203				goto del_dev;
4204
4205			macsec->insert_tx_tag =
4206				macsec_needs_tx_tag(macsec, ops);
4207		}
4208	}
4209
4210	err = register_macsec_dev(real_dev, dev);
4211	if (err < 0)
4212		goto del_dev;
4213
4214	netdev_update_features(dev);
4215	netif_stacked_transfer_operstate(real_dev, dev);
4216	linkwatch_fire_event(dev);
4217
4218	macsec_generation++;
4219
4220	return 0;
4221
4222del_dev:
4223	macsec_del_dev(macsec);
4224unlink:
4225	netdev_upper_dev_unlink(real_dev, dev);
4226unregister:
4227	unregister_netdevice(dev);
4228	return err;
4229}
4230
4231static int validate_cipher_suite(const struct nlattr *attr,
4232				 struct netlink_ext_ack *extack)
4233{
4234	switch (nla_get_u64(attr)) {
4235	case MACSEC_CIPHER_ID_GCM_AES_128:
4236	case MACSEC_CIPHER_ID_GCM_AES_256:
4237	case MACSEC_CIPHER_ID_GCM_AES_XPN_128:
4238	case MACSEC_CIPHER_ID_GCM_AES_XPN_256:
4239	case MACSEC_DEFAULT_CIPHER_ID:
4240		return 0;
4241	default:
4242		return -EINVAL;
4243	}
4244}
4245
4246static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
4247				struct netlink_ext_ack *extack)
4248{
4249	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
4250	bool es, scb, sci;
4251
4252	if (!data)
4253		return 0;
4254
4255	if (data[IFLA_MACSEC_ICV_LEN]) {
4256		icv_len = nla_get_u8(data[IFLA_MACSEC_ICV_LEN]);
4257		if (icv_len != MACSEC_DEFAULT_ICV_LEN) {
4258			char dummy_key[DEFAULT_SAK_LEN] = { 0 };
4259			struct crypto_aead *dummy_tfm;
4260
4261			dummy_tfm = macsec_alloc_tfm(dummy_key,
4262						     DEFAULT_SAK_LEN,
4263						     icv_len);
4264			if (IS_ERR(dummy_tfm))
4265				return PTR_ERR(dummy_tfm);
4266			crypto_free_aead(dummy_tfm);
4267		}
4268	}
4269
4270	es  = nla_get_u8_default(data[IFLA_MACSEC_ES], false);
4271	sci = nla_get_u8_default(data[IFLA_MACSEC_INC_SCI], false);
4272	scb = nla_get_u8_default(data[IFLA_MACSEC_SCB], false);
4273
4274	if ((sci && (scb || es)) || (scb && es))
4275		return -EINVAL;
4276
4277	if ((data[IFLA_MACSEC_REPLAY_PROTECT] &&
4278	     nla_get_u8(data[IFLA_MACSEC_REPLAY_PROTECT])) &&
4279	    !data[IFLA_MACSEC_WINDOW])
4280		return -EINVAL;
4281
4282	return 0;
4283}
4284
4285static struct net *macsec_get_link_net(const struct net_device *dev)
4286{
4287	return dev_net(macsec_priv(dev)->real_dev);
4288}
4289
4290struct net_device *macsec_get_real_dev(const struct net_device *dev)
4291{
4292	return macsec_priv(dev)->real_dev;
4293}
4294EXPORT_SYMBOL_GPL(macsec_get_real_dev);
4295
4296bool macsec_netdev_is_offloaded(struct net_device *dev)
4297{
4298	return macsec_is_offloaded(macsec_priv(dev));
4299}
4300EXPORT_SYMBOL_GPL(macsec_netdev_is_offloaded);
4301
4302static size_t macsec_get_size(const struct net_device *dev)
4303{
4304	return  nla_total_size_64bit(8) + /* IFLA_MACSEC_SCI */
4305		nla_total_size(1) + /* IFLA_MACSEC_ICV_LEN */
4306		nla_total_size_64bit(8) + /* IFLA_MACSEC_CIPHER_SUITE */
4307		nla_total_size(4) + /* IFLA_MACSEC_WINDOW */
4308		nla_total_size(1) + /* IFLA_MACSEC_ENCODING_SA */
4309		nla_total_size(1) + /* IFLA_MACSEC_ENCRYPT */
4310		nla_total_size(1) + /* IFLA_MACSEC_PROTECT */
4311		nla_total_size(1) + /* IFLA_MACSEC_INC_SCI */
4312		nla_total_size(1) + /* IFLA_MACSEC_ES */
4313		nla_total_size(1) + /* IFLA_MACSEC_SCB */
4314		nla_total_size(1) + /* IFLA_MACSEC_REPLAY_PROTECT */
4315		nla_total_size(1) + /* IFLA_MACSEC_VALIDATION */
4316		nla_total_size(1) + /* IFLA_MACSEC_OFFLOAD */
4317		0;
4318}
4319
4320static int macsec_fill_info(struct sk_buff *skb,
4321			    const struct net_device *dev)
4322{
4323	struct macsec_tx_sc *tx_sc;
4324	struct macsec_dev *macsec;
4325	struct macsec_secy *secy;
4326	u64 csid;
4327
4328	macsec = macsec_priv(dev);
4329	secy = &macsec->secy;
4330	tx_sc = &secy->tx_sc;
4331
4332	switch (secy->key_len) {
4333	case MACSEC_GCM_AES_128_SAK_LEN:
4334		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_128 : MACSEC_DEFAULT_CIPHER_ID;
4335		break;
4336	case MACSEC_GCM_AES_256_SAK_LEN:
4337		csid = secy->xpn ? MACSEC_CIPHER_ID_GCM_AES_XPN_256 : MACSEC_CIPHER_ID_GCM_AES_256;
4338		break;
4339	default:
4340		goto nla_put_failure;
4341	}
4342
4343	if (nla_put_sci(skb, IFLA_MACSEC_SCI, secy->sci,
4344			IFLA_MACSEC_PAD) ||
4345	    nla_put_u8(skb, IFLA_MACSEC_ICV_LEN, secy->icv_len) ||
4346	    nla_put_u64_64bit(skb, IFLA_MACSEC_CIPHER_SUITE,
4347			      csid, IFLA_MACSEC_PAD) ||
4348	    nla_put_u8(skb, IFLA_MACSEC_ENCODING_SA, tx_sc->encoding_sa) ||
4349	    nla_put_u8(skb, IFLA_MACSEC_ENCRYPT, tx_sc->encrypt) ||
4350	    nla_put_u8(skb, IFLA_MACSEC_PROTECT, secy->protect_frames) ||
4351	    nla_put_u8(skb, IFLA_MACSEC_INC_SCI, tx_sc->send_sci) ||
4352	    nla_put_u8(skb, IFLA_MACSEC_ES, tx_sc->end_station) ||
4353	    nla_put_u8(skb, IFLA_MACSEC_SCB, tx_sc->scb) ||
4354	    nla_put_u8(skb, IFLA_MACSEC_REPLAY_PROTECT, secy->replay_protect) ||
4355	    nla_put_u8(skb, IFLA_MACSEC_VALIDATION, secy->validate_frames) ||
4356	    nla_put_u8(skb, IFLA_MACSEC_OFFLOAD, macsec->offload) ||
4357	    0)
4358		goto nla_put_failure;
4359
4360	if (secy->replay_protect) {
4361		if (nla_put_u32(skb, IFLA_MACSEC_WINDOW, secy->replay_window))
4362			goto nla_put_failure;
4363	}
4364
4365	return 0;
4366
4367nla_put_failure:
4368	return -EMSGSIZE;
4369}
4370
4371static struct rtnl_link_ops macsec_link_ops __read_mostly = {
4372	.kind		= "macsec",
4373	.priv_size	= sizeof(struct macsec_dev),
4374	.maxtype	= IFLA_MACSEC_MAX,
4375	.policy		= macsec_rtnl_policy,
4376	.setup		= macsec_setup,
4377	.validate	= macsec_validate_attr,
4378	.newlink	= macsec_newlink,
4379	.changelink	= macsec_changelink,
4380	.dellink	= macsec_dellink,
4381	.get_size	= macsec_get_size,
4382	.fill_info	= macsec_fill_info,
4383	.get_link_net	= macsec_get_link_net,
4384};
4385
4386static bool is_macsec_master(struct net_device *dev)
4387{
4388	return rcu_access_pointer(dev->rx_handler) == macsec_handle_frame;
4389}
4390
4391static int macsec_notify(struct notifier_block *this, unsigned long event,
4392			 void *ptr)
4393{
4394	struct net_device *real_dev = netdev_notifier_info_to_dev(ptr);
4395	struct macsec_rxh_data *rxd;
4396	struct macsec_dev *m, *n;
4397	LIST_HEAD(head);
4398
4399	if (!is_macsec_master(real_dev))
4400		return NOTIFY_DONE;
4401
4402	rxd = macsec_data_rtnl(real_dev);
4403
4404	switch (event) {
4405	case NETDEV_DOWN:
4406	case NETDEV_UP:
4407	case NETDEV_CHANGE:
4408		list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4409			struct net_device *dev = m->secy.netdev;
4410
4411			netif_stacked_transfer_operstate(real_dev, dev);
4412		}
4413		break;
4414	case NETDEV_UNREGISTER:
4415		list_for_each_entry_safe(m, n, &rxd->secys, secys) {
4416			macsec_common_dellink(m->secy.netdev, &head);
4417		}
4418
4419		netdev_rx_handler_unregister(real_dev);
4420		kfree(rxd);
4421
4422		unregister_netdevice_many(&head);
4423		break;
4424	case NETDEV_CHANGEMTU:
4425		list_for_each_entry(m, &rxd->secys, secys) {
4426			struct net_device *dev = m->secy.netdev;
4427			unsigned int mtu = real_dev->mtu - (m->secy.icv_len +
4428							    macsec_extra_len(true));
4429
4430			if (dev->mtu > mtu)
4431				dev_set_mtu(dev, mtu);
4432		}
4433		break;
4434	case NETDEV_FEAT_CHANGE:
4435		list_for_each_entry(m, &rxd->secys, secys) {
4436			macsec_inherit_tso_max(m->secy.netdev);
4437			netdev_update_features(m->secy.netdev);
4438		}
4439		break;
4440	}
4441
4442	return NOTIFY_OK;
4443}
4444
4445static struct notifier_block macsec_notifier = {
4446	.notifier_call = macsec_notify,
4447};
4448
4449static int __init macsec_init(void)
4450{
4451	int err;
4452
4453	pr_info("MACsec IEEE 802.1AE\n");
4454	err = register_netdevice_notifier(&macsec_notifier);
4455	if (err)
4456		return err;
4457
4458	err = rtnl_link_register(&macsec_link_ops);
4459	if (err)
4460		goto notifier;
4461
4462	err = genl_register_family(&macsec_fam);
4463	if (err)
4464		goto rtnl;
4465
4466	return 0;
4467
4468rtnl:
4469	rtnl_link_unregister(&macsec_link_ops);
4470notifier:
4471	unregister_netdevice_notifier(&macsec_notifier);
4472	return err;
4473}
4474
4475static void __exit macsec_exit(void)
4476{
4477	genl_unregister_family(&macsec_fam);
4478	rtnl_link_unregister(&macsec_link_ops);
4479	unregister_netdevice_notifier(&macsec_notifier);
4480	rcu_barrier();
4481}
4482
4483module_init(macsec_init);
4484module_exit(macsec_exit);
4485
4486MODULE_ALIAS_RTNL_LINK("macsec");
4487MODULE_ALIAS_GENL_FAMILY("macsec");
4488
4489MODULE_DESCRIPTION("MACsec IEEE 802.1AE");
4490MODULE_LICENSE("GPL v2");