crypto.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * net/tipc/crypto.c: TIPC crypto for key handling & packet en/decryption
   4 *
   5 * Copyright (c) 2019, Ericsson AB
   6 * All rights reserved.
   7 *
   8 * Redistribution and use in source and binary forms, with or without
   9 * modification, are permitted provided that the following conditions are met:
  10 *
  11 * 1. Redistributions of source code must retain the above copyright
  12 *    notice, this list of conditions and the following disclaimer.
  13 * 2. Redistributions in binary form must reproduce the above copyright
  14 *    notice, this list of conditions and the following disclaimer in the
  15 *    documentation and/or other materials provided with the distribution.
  16 * 3. Neither the names of the copyright holders nor the names of its
  17 *    contributors may be used to endorse or promote products derived from
  18 *    this software without specific prior written permission.
  19 *
  20 * Alternatively, this software may be distributed under the terms of the
  21 * GNU General Public License ("GPL") version 2 as published by the Free
  22 * Software Foundation.
  23 *
  24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34 * POSSIBILITY OF SUCH DAMAGE.
  35 */
  36
  37#include <crypto/aead.h>
  38#include <crypto/aes.h>
  39#include <crypto/rng.h>
  40#include "crypto.h"
  41#include "msg.h"
  42#include "bcast.h"
  43
  44#define TIPC_TX_GRACE_PERIOD	msecs_to_jiffies(5000) /* 5s */
  45#define TIPC_TX_LASTING_TIME	msecs_to_jiffies(10000) /* 10s */
  46#define TIPC_RX_ACTIVE_LIM	msecs_to_jiffies(3000) /* 3s */
  47#define TIPC_RX_PASSIVE_LIM	msecs_to_jiffies(15000) /* 15s */
  48
  49#define TIPC_MAX_TFMS_DEF	10
  50#define TIPC_MAX_TFMS_LIM	1000
  51
  52#define TIPC_REKEYING_INTV_DEF	(60 * 24) /* default: 1 day */
  53
  54/*
  55 * TIPC Key ids
  56 */
  57enum {
  58	KEY_MASTER = 0,
  59	KEY_MIN = KEY_MASTER,
  60	KEY_1 = 1,
  61	KEY_2,
  62	KEY_3,
  63	KEY_MAX = KEY_3,
  64};
  65
  66/*
  67 * TIPC Crypto statistics
  68 */
  69enum {
  70	STAT_OK,
  71	STAT_NOK,
  72	STAT_ASYNC,
  73	STAT_ASYNC_OK,
  74	STAT_ASYNC_NOK,
  75	STAT_BADKEYS, /* tx only */
  76	STAT_BADMSGS = STAT_BADKEYS, /* rx only */
  77	STAT_NOKEYS,
  78	STAT_SWITCHES,
  79
  80	MAX_STATS,
  81};
  82
  83/* TIPC crypto statistics' header */
  84static const char *hstats[MAX_STATS] = {"ok", "nok", "async", "async_ok",
  85					"async_nok", "badmsgs", "nokeys",
  86					"switches"};
  87
  88/* Max TFMs number per key */
  89int sysctl_tipc_max_tfms __read_mostly = TIPC_MAX_TFMS_DEF;
  90/* Key exchange switch, default: on */
  91int sysctl_tipc_key_exchange_enabled __read_mostly = 1;
  92
  93/*
  94 * struct tipc_key - TIPC keys' status indicator
  95 *
  96 *         7     6     5     4     3     2     1     0
  97 *      +-----+-----+-----+-----+-----+-----+-----+-----+
  98 * key: | (reserved)|passive idx| active idx|pending idx|
  99 *      +-----+-----+-----+-----+-----+-----+-----+-----+
 100 */
 101struct tipc_key {
 102#define KEY_BITS (2)
 103#define KEY_MASK ((1 << KEY_BITS) - 1)
 104	union {
 105		struct {
 106#if defined(__LITTLE_ENDIAN_BITFIELD)
 107			u8 pending:2,
 108			   active:2,
 109			   passive:2, /* rx only */
 110			   reserved:2;
 111#elif defined(__BIG_ENDIAN_BITFIELD)
 112			u8 reserved:2,
 113			   passive:2, /* rx only */
 114			   active:2,
 115			   pending:2;
 116#else
 117#error  "Please fix <asm/byteorder.h>"
 118#endif
 119		} __packed;
 120		u8 keys;
 121	};
 122};
 123
 124/**
 125 * struct tipc_tfm - TIPC TFM structure to form a list of TFMs
 126 * @tfm: cipher handle/key
 127 * @list: linked list of TFMs
 128 */
 129struct tipc_tfm {
 130	struct crypto_aead *tfm;
 131	struct list_head list;
 132};
 133
 134/**
 135 * struct tipc_aead - TIPC AEAD key structure
 136 * @tfm_entry: per-cpu pointer to one entry in TFM list
 137 * @crypto: TIPC crypto owns this key
 138 * @cloned: reference to the source key in case cloning
 139 * @users: the number of the key users (TX/RX)
 140 * @salt: the key's SALT value
 141 * @authsize: authentication tag size (max = 16)
 142 * @mode: crypto mode is applied to the key
 143 * @hint: a hint for user key
 144 * @rcu: struct rcu_head
 145 * @key: the aead key
 146 * @gen: the key's generation
 147 * @seqno: the key seqno (cluster scope)
 148 * @refcnt: the key reference counter
 149 */
 150struct tipc_aead {
 151#define TIPC_AEAD_HINT_LEN (5)
 152	struct tipc_tfm * __percpu *tfm_entry;
 153	struct tipc_crypto *crypto;
 154	struct tipc_aead *cloned;
 155	atomic_t users;
 156	u32 salt;
 157	u8 authsize;
 158	u8 mode;
 159	char hint[2 * TIPC_AEAD_HINT_LEN + 1];
 160	struct rcu_head rcu;
 161	struct tipc_aead_key *key;
 162	u16 gen;
 163
 164	atomic64_t seqno ____cacheline_aligned;
 165	refcount_t refcnt ____cacheline_aligned;
 166
 167} ____cacheline_aligned;
 168
 169/**
 170 * struct tipc_crypto_stats - TIPC Crypto statistics
 171 * @stat: array of crypto statistics
 172 */
 173struct tipc_crypto_stats {
 174	unsigned int stat[MAX_STATS];
 175};
 176
 177/**
 178 * struct tipc_crypto - TIPC TX/RX crypto structure
 179 * @net: struct net
 180 * @node: TIPC node (RX)
 181 * @aead: array of pointers to AEAD keys for encryption/decryption
 182 * @peer_rx_active: replicated peer RX active key index
 183 * @key_gen: TX/RX key generation
 184 * @key: the key states
 185 * @skey_mode: session key's mode
 186 * @skey: received session key
 187 * @wq: common workqueue on TX crypto
 188 * @work: delayed work sched for TX/RX
 189 * @key_distr: key distributing state
 190 * @rekeying_intv: rekeying interval (in minutes)
 191 * @stats: the crypto statistics
 192 * @name: the crypto name
 193 * @sndnxt: the per-peer sndnxt (TX)
 194 * @timer1: general timer 1 (jiffies)
 195 * @timer2: general timer 2 (jiffies)
 196 * @working: the crypto is working or not
 197 * @key_master: flag indicates if master key exists
 198 * @legacy_user: flag indicates if a peer joins w/o master key (for bwd comp.)
 199 * @nokey: no key indication
 200 * @flags: combined flags field
 201 * @lock: tipc_key lock
 202 */
 203struct tipc_crypto {
 204	struct net *net;
 205	struct tipc_node *node;
 206	struct tipc_aead __rcu *aead[KEY_MAX + 1];
 207	atomic_t peer_rx_active;
 208	u16 key_gen;
 209	struct tipc_key key;
 210	u8 skey_mode;
 211	struct tipc_aead_key *skey;
 212	struct workqueue_struct *wq;
 213	struct delayed_work work;
 214#define KEY_DISTR_SCHED		1
 215#define KEY_DISTR_COMPL		2
 216	atomic_t key_distr;
 217	u32 rekeying_intv;
 218
 219	struct tipc_crypto_stats __percpu *stats;
 220	char name[48];
 221
 222	atomic64_t sndnxt ____cacheline_aligned;
 223	unsigned long timer1;
 224	unsigned long timer2;
 225	union {
 226		struct {
 227			u8 working:1;
 228			u8 key_master:1;
 229			u8 legacy_user:1;
 230			u8 nokey: 1;
 231		};
 232		u8 flags;
 233	};
 234	spinlock_t lock; /* crypto lock */
 235
 236} ____cacheline_aligned;
 237
 238/* struct tipc_crypto_tx_ctx - TX context for callbacks */
 239struct tipc_crypto_tx_ctx {
 240	struct tipc_aead *aead;
 241	struct tipc_bearer *bearer;
 242	struct tipc_media_addr dst;
 243};
 244
 245/* struct tipc_crypto_rx_ctx - RX context for callbacks */
 246struct tipc_crypto_rx_ctx {
 247	struct tipc_aead *aead;
 248	struct tipc_bearer *bearer;
 249};
 250
 251static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead);
 252static inline void tipc_aead_put(struct tipc_aead *aead);
 253static void tipc_aead_free(struct rcu_head *rp);
 254static int tipc_aead_users(struct tipc_aead __rcu *aead);
 255static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim);
 256static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim);
 257static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val);
 258static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead);
 259static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
 260			  u8 mode);
 261static int tipc_aead_clone(struct tipc_aead **dst, struct tipc_aead *src);
 262static void *tipc_aead_mem_alloc(struct crypto_aead *tfm,
 263				 unsigned int crypto_ctx_size,
 264				 u8 **iv, struct aead_request **req,
 265				 struct scatterlist **sg, int nsg);
 266static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
 267			     struct tipc_bearer *b,
 268			     struct tipc_media_addr *dst,
 269			     struct tipc_node *__dnode);
 270static void tipc_aead_encrypt_done(void *data, int err);
 271static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
 272			     struct sk_buff *skb, struct tipc_bearer *b);
 273static void tipc_aead_decrypt_done(void *data, int err);
 274static inline int tipc_ehdr_size(struct tipc_ehdr *ehdr);
 275static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
 276			   u8 tx_key, struct sk_buff *skb,
 277			   struct tipc_crypto *__rx);
 278static inline void tipc_crypto_key_set_state(struct tipc_crypto *c,
 279					     u8 new_passive,
 280					     u8 new_active,
 281					     u8 new_pending);
 282static int tipc_crypto_key_attach(struct tipc_crypto *c,
 283				  struct tipc_aead *aead, u8 pos,
 284				  bool master_key);
 285static bool tipc_crypto_key_try_align(struct tipc_crypto *rx, u8 new_pending);
 286static struct tipc_aead *tipc_crypto_key_pick_tx(struct tipc_crypto *tx,
 287						 struct tipc_crypto *rx,
 288						 struct sk_buff *skb,
 289						 u8 tx_key);
 290static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb);
 291static int tipc_crypto_key_revoke(struct net *net, u8 tx_key);
 292static inline void tipc_crypto_clone_msg(struct net *net, struct sk_buff *_skb,
 293					 struct tipc_bearer *b,
 294					 struct tipc_media_addr *dst,
 295					 struct tipc_node *__dnode, u8 type);
 296static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
 297				     struct tipc_bearer *b,
 298				     struct sk_buff **skb, int err);
 299static void tipc_crypto_do_cmd(struct net *net, int cmd);
 300static char *tipc_crypto_key_dump(struct tipc_crypto *c, char *buf);
 301static char *tipc_key_change_dump(struct tipc_key old, struct tipc_key new,
 302				  char *buf);
 303static int tipc_crypto_key_xmit(struct net *net, struct tipc_aead_key *skey,
 304				u16 gen, u8 mode, u32 dnode);
 305static bool tipc_crypto_key_rcv(struct tipc_crypto *rx, struct tipc_msg *hdr);
 306static void tipc_crypto_work_tx(struct work_struct *work);
 307static void tipc_crypto_work_rx(struct work_struct *work);
 308static int tipc_aead_key_generate(struct tipc_aead_key *skey);
 309
 310#define is_tx(crypto) (!(crypto)->node)
 311#define is_rx(crypto) (!is_tx(crypto))
 312
 313#define key_next(cur) ((cur) % KEY_MAX + 1)
 314
 315#define tipc_aead_rcu_ptr(rcu_ptr, lock)				\
 316	rcu_dereference_protected((rcu_ptr), lockdep_is_held(lock))
 317
 318#define tipc_aead_rcu_replace(rcu_ptr, ptr, lock)			\
 319do {									\
 320	struct tipc_aead *__tmp = rcu_dereference_protected((rcu_ptr),	\
 321						lockdep_is_held(lock));	\
 322	rcu_assign_pointer((rcu_ptr), (ptr));				\
 323	tipc_aead_put(__tmp);						\
 324} while (0)
 325
 326#define tipc_crypto_key_detach(rcu_ptr, lock)				\
 327	tipc_aead_rcu_replace((rcu_ptr), NULL, lock)
 328
 329/**
 330 * tipc_aead_key_validate - Validate a AEAD user key
 331 * @ukey: pointer to user key data
 332 * @info: netlink info pointer
 333 */
 334int tipc_aead_key_validate(struct tipc_aead_key *ukey, struct genl_info *info)
 335{
 336	int keylen;
 337
 338	/* Check if algorithm exists */
 339	if (unlikely(!crypto_has_alg(ukey->alg_name, 0, 0))) {
 340		GENL_SET_ERR_MSG(info, "unable to load the algorithm (module existed?)");
 341		return -ENODEV;
 342	}
 343
 344	/* Currently, we only support the "gcm(aes)" cipher algorithm */
 345	if (strcmp(ukey->alg_name, "gcm(aes)")) {
 346		GENL_SET_ERR_MSG(info, "not supported yet the algorithm");
 347		return -ENOTSUPP;
 348	}
 349
 350	/* Check if key size is correct */
 351	keylen = ukey->keylen - TIPC_AES_GCM_SALT_SIZE;
 352	if (unlikely(keylen != TIPC_AES_GCM_KEY_SIZE_128 &&
 353		     keylen != TIPC_AES_GCM_KEY_SIZE_192 &&
 354		     keylen != TIPC_AES_GCM_KEY_SIZE_256)) {
 355		GENL_SET_ERR_MSG(info, "incorrect key length (20, 28 or 36 octets?)");
 356		return -EKEYREJECTED;
 357	}
 358
 359	return 0;
 360}
 361
 362/**
 363 * tipc_aead_key_generate - Generate new session key
 364 * @skey: input/output key with new content
 365 *
 366 * Return: 0 in case of success, otherwise < 0
 367 */
 368static int tipc_aead_key_generate(struct tipc_aead_key *skey)
 369{
 370	int rc = 0;
 371
 372	/* Fill the key's content with a random value via RNG cipher */
 373	rc = crypto_get_default_rng();
 374	if (likely(!rc)) {
 375		rc = crypto_rng_get_bytes(crypto_default_rng, skey->key,
 376					  skey->keylen);
 377		crypto_put_default_rng();
 378	}
 379
 380	return rc;
 381}
 382
 383static struct tipc_aead *tipc_aead_get(struct tipc_aead __rcu *aead)
 384{
 385	struct tipc_aead *tmp;
 386
 387	rcu_read_lock();
 388	tmp = rcu_dereference(aead);
 389	if (unlikely(!tmp || !refcount_inc_not_zero(&tmp->refcnt)))
 390		tmp = NULL;
 391	rcu_read_unlock();
 392
 393	return tmp;
 394}
 395
 396static inline void tipc_aead_put(struct tipc_aead *aead)
 397{
 398	if (aead && refcount_dec_and_test(&aead->refcnt))
 399		call_rcu(&aead->rcu, tipc_aead_free);
 400}
 401
 402/**
 403 * tipc_aead_free - Release AEAD key incl. all the TFMs in the list
 404 * @rp: rcu head pointer
 405 */
 406static void tipc_aead_free(struct rcu_head *rp)
 407{
 408	struct tipc_aead *aead = container_of(rp, struct tipc_aead, rcu);
 409	struct tipc_tfm *tfm_entry, *head, *tmp;
 410
 411	if (aead->cloned) {
 412		tipc_aead_put(aead->cloned);
 413	} else {
 414		head = *get_cpu_ptr(aead->tfm_entry);
 415		put_cpu_ptr(aead->tfm_entry);
 416		list_for_each_entry_safe(tfm_entry, tmp, &head->list, list) {
 417			crypto_free_aead(tfm_entry->tfm);
 418			list_del(&tfm_entry->list);
 419			kfree(tfm_entry);
 420		}
 421		/* Free the head */
 422		crypto_free_aead(head->tfm);
 423		list_del(&head->list);
 424		kfree(head);
 425	}
 426	free_percpu(aead->tfm_entry);
 427	kfree_sensitive(aead->key);
 428	kfree_sensitive(aead);
 429}
 430
 431static int tipc_aead_users(struct tipc_aead __rcu *aead)
 432{
 433	struct tipc_aead *tmp;
 434	int users = 0;
 435
 436	rcu_read_lock();
 437	tmp = rcu_dereference(aead);
 438	if (tmp)
 439		users = atomic_read(&tmp->users);
 440	rcu_read_unlock();
 441
 442	return users;
 443}
 444
 445static void tipc_aead_users_inc(struct tipc_aead __rcu *aead, int lim)
 446{
 447	struct tipc_aead *tmp;
 448
 449	rcu_read_lock();
 450	tmp = rcu_dereference(aead);
 451	if (tmp)
 452		atomic_add_unless(&tmp->users, 1, lim);
 453	rcu_read_unlock();
 454}
 455
 456static void tipc_aead_users_dec(struct tipc_aead __rcu *aead, int lim)
 457{
 458	struct tipc_aead *tmp;
 459
 460	rcu_read_lock();
 461	tmp = rcu_dereference(aead);
 462	if (tmp)
 463		atomic_add_unless(&tmp->users, -1, lim);
 464	rcu_read_unlock();
 465}
 466
 467static void tipc_aead_users_set(struct tipc_aead __rcu *aead, int val)
 468{
 469	struct tipc_aead *tmp;
 470	int cur;
 471
 472	rcu_read_lock();
 473	tmp = rcu_dereference(aead);
 474	if (tmp) {
 475		do {
 476			cur = atomic_read(&tmp->users);
 477			if (cur == val)
 478				break;
 479		} while (atomic_cmpxchg(&tmp->users, cur, val) != cur);
 480	}
 481	rcu_read_unlock();
 482}
 483
 484/**
 485 * tipc_aead_tfm_next - Move TFM entry to the next one in list and return it
 486 * @aead: the AEAD key pointer
 487 */
 488static struct crypto_aead *tipc_aead_tfm_next(struct tipc_aead *aead)
 489{
 490	struct tipc_tfm **tfm_entry;
 491	struct crypto_aead *tfm;
 492
 493	tfm_entry = get_cpu_ptr(aead->tfm_entry);
 494	*tfm_entry = list_next_entry(*tfm_entry, list);
 495	tfm = (*tfm_entry)->tfm;
 496	put_cpu_ptr(tfm_entry);
 497
 498	return tfm;
 499}
 500
 501/**
 502 * tipc_aead_init - Initiate TIPC AEAD
 503 * @aead: returned new TIPC AEAD key handle pointer
 504 * @ukey: pointer to user key data
 505 * @mode: the key mode
 506 *
 507 * Allocate a (list of) new cipher transformation (TFM) with the specific user
 508 * key data if valid. The number of the allocated TFMs can be set via the sysfs
 509 * "net/tipc/max_tfms" first.
 510 * Also, all the other AEAD data are also initialized.
 511 *
 512 * Return: 0 if the initiation is successful, otherwise: < 0
 513 */
 514static int tipc_aead_init(struct tipc_aead **aead, struct tipc_aead_key *ukey,
 515			  u8 mode)
 516{
 517	struct tipc_tfm *tfm_entry, *head;
 518	struct crypto_aead *tfm;
 519	struct tipc_aead *tmp;
 520	int keylen, err, cpu;
 521	int tfm_cnt = 0;
 522
 523	if (unlikely(*aead))
 524		return -EEXIST;
 525
 526	/* Allocate a new AEAD */
 527	tmp = kzalloc(sizeof(*tmp), GFP_ATOMIC);
 528	if (unlikely(!tmp))
 529		return -ENOMEM;
 530
 531	/* The key consists of two parts: [AES-KEY][SALT] */
 532	keylen = ukey->keylen - TIPC_AES_GCM_SALT_SIZE;
 533
 534	/* Allocate per-cpu TFM entry pointer */
 535	tmp->tfm_entry = alloc_percpu(struct tipc_tfm *);
 536	if (!tmp->tfm_entry) {
 537		kfree_sensitive(tmp);
 538		return -ENOMEM;
 539	}
 540
 541	/* Make a list of TFMs with the user key data */
 542	do {
 543		tfm = crypto_alloc_aead(ukey->alg_name, 0, 0);
 544		if (IS_ERR(tfm)) {
 545			err = PTR_ERR(tfm);
 546			break;
 547		}
 548
 549		if (unlikely(!tfm_cnt &&
 550			     crypto_aead_ivsize(tfm) != TIPC_AES_GCM_IV_SIZE)) {
 551			crypto_free_aead(tfm);
 552			err = -ENOTSUPP;
 553			break;
 554		}
 555
 556		err = crypto_aead_setauthsize(tfm, TIPC_AES_GCM_TAG_SIZE);
 557		err |= crypto_aead_setkey(tfm, ukey->key, keylen);
 558		if (unlikely(err)) {
 559			crypto_free_aead(tfm);
 560			break;
 561		}
 562
 563		tfm_entry = kmalloc(sizeof(*tfm_entry), GFP_KERNEL);
 564		if (unlikely(!tfm_entry)) {
 565			crypto_free_aead(tfm);
 566			err = -ENOMEM;
 567			break;
 568		}
 569		INIT_LIST_HEAD(&tfm_entry->list);
 570		tfm_entry->tfm = tfm;
 571
 572		/* First entry? */
 573		if (!tfm_cnt) {
 574			head = tfm_entry;
 575			for_each_possible_cpu(cpu) {
 576				*per_cpu_ptr(tmp->tfm_entry, cpu) = head;
 577			}
 578		} else {
 579			list_add_tail(&tfm_entry->list, &head->list);
 580		}
 581
 582	} while (++tfm_cnt < sysctl_tipc_max_tfms);
 583
 584	/* Not any TFM is allocated? */
 585	if (!tfm_cnt) {
 586		free_percpu(tmp->tfm_entry);
 587		kfree_sensitive(tmp);
 588		return err;
 589	}
 590
 591	/* Form a hex string of some last bytes as the key's hint */
 592	bin2hex(tmp->hint, ukey->key + keylen - TIPC_AEAD_HINT_LEN,
 593		TIPC_AEAD_HINT_LEN);
 594
 595	/* Initialize the other data */
 596	tmp->mode = mode;
 597	tmp->cloned = NULL;
 598	tmp->authsize = TIPC_AES_GCM_TAG_SIZE;
 599	tmp->key = kmemdup(ukey, tipc_aead_key_size(ukey), GFP_KERNEL);
 600	if (!tmp->key) {
 601		tipc_aead_free(&tmp->rcu);
 602		return -ENOMEM;
 603	}
 604	memcpy(&tmp->salt, ukey->key + keylen, TIPC_AES_GCM_SALT_SIZE);
 605	atomic_set(&tmp->users, 0);
 606	atomic64_set(&tmp->seqno, 0);
 607	refcount_set(&tmp->refcnt, 1);
 608
 609	*aead = tmp;
 610	return 0;
 611}
 612
 613/**
 614 * tipc_aead_clone - Clone a TIPC AEAD key
 615 * @dst: dest key for the cloning
 616 * @src: source key to clone from
 617 *
 618 * Make a "copy" of the source AEAD key data to the dest, the TFMs list is
 619 * common for the keys.
 620 * A reference to the source is hold in the "cloned" pointer for the later
 621 * freeing purposes.
 622 *
 623 * Note: this must be done in cluster-key mode only!
 624 * Return: 0 in case of success, otherwise < 0
 625 */
 626static int tipc_aead_clone(struct tipc_aead **dst, struct tipc_aead *src)
 627{
 628	struct tipc_aead *aead;
 629	int cpu;
 630
 631	if (!src)
 632		return -ENOKEY;
 633
 634	if (src->mode != CLUSTER_KEY)
 635		return -EINVAL;
 636
 637	if (unlikely(*dst))
 638		return -EEXIST;
 639
 640	aead = kzalloc(sizeof(*aead), GFP_ATOMIC);
 641	if (unlikely(!aead))
 642		return -ENOMEM;
 643
 644	aead->tfm_entry = alloc_percpu_gfp(struct tipc_tfm *, GFP_ATOMIC);
 645	if (unlikely(!aead->tfm_entry)) {
 646		kfree_sensitive(aead);
 647		return -ENOMEM;
 648	}
 649
 650	for_each_possible_cpu(cpu) {
 651		*per_cpu_ptr(aead->tfm_entry, cpu) =
 652				*per_cpu_ptr(src->tfm_entry, cpu);
 653	}
 654
 655	memcpy(aead->hint, src->hint, sizeof(src->hint));
 656	aead->mode = src->mode;
 657	aead->salt = src->salt;
 658	aead->authsize = src->authsize;
 659	atomic_set(&aead->users, 0);
 660	atomic64_set(&aead->seqno, 0);
 661	refcount_set(&aead->refcnt, 1);
 662
 663	WARN_ON(!refcount_inc_not_zero(&src->refcnt));
 664	aead->cloned = src;
 665
 666	*dst = aead;
 667	return 0;
 668}
 669
 670/**
 671 * tipc_aead_mem_alloc - Allocate memory for AEAD request operations
 672 * @tfm: cipher handle to be registered with the request
 673 * @crypto_ctx_size: size of crypto context for callback
 674 * @iv: returned pointer to IV data
 675 * @req: returned pointer to AEAD request data
 676 * @sg: returned pointer to SG lists
 677 * @nsg: number of SG lists to be allocated
 678 *
 679 * Allocate memory to store the crypto context data, AEAD request, IV and SG
 680 * lists, the memory layout is as follows:
 681 * crypto_ctx || iv || aead_req || sg[]
 682 *
 683 * Return: the pointer to the memory areas in case of success, otherwise NULL
 684 */
 685static void *tipc_aead_mem_alloc(struct crypto_aead *tfm,
 686				 unsigned int crypto_ctx_size,
 687				 u8 **iv, struct aead_request **req,
 688				 struct scatterlist **sg, int nsg)
 689{
 690	unsigned int iv_size, req_size;
 691	unsigned int len;
 692	u8 *mem;
 693
 694	iv_size = crypto_aead_ivsize(tfm);
 695	req_size = sizeof(**req) + crypto_aead_reqsize(tfm);
 696
 697	len = crypto_ctx_size;
 698	len += iv_size;
 699	len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1);
 700	len = ALIGN(len, crypto_tfm_ctx_alignment());
 701	len += req_size;
 702	len = ALIGN(len, __alignof__(struct scatterlist));
 703	len += nsg * sizeof(**sg);
 704
 705	mem = kmalloc(len, GFP_ATOMIC);
 706	if (!mem)
 707		return NULL;
 708
 709	*iv = (u8 *)PTR_ALIGN(mem + crypto_ctx_size,
 710			      crypto_aead_alignmask(tfm) + 1);
 711	*req = (struct aead_request *)PTR_ALIGN(*iv + iv_size,
 712						crypto_tfm_ctx_alignment());
 713	*sg = (struct scatterlist *)PTR_ALIGN((u8 *)*req + req_size,
 714					      __alignof__(struct scatterlist));
 715
 716	return (void *)mem;
 717}
 718
 719/**
 720 * tipc_aead_encrypt - Encrypt a message
 721 * @aead: TIPC AEAD key for the message encryption
 722 * @skb: the input/output skb
 723 * @b: TIPC bearer where the message will be delivered after the encryption
 724 * @dst: the destination media address
 725 * @__dnode: TIPC dest node if "known"
 726 *
 727 * Return:
 728 * * 0                   : if the encryption has completed
 729 * * -EINPROGRESS/-EBUSY : if a callback will be performed
 730 * * < 0                 : the encryption has failed
 731 */
 732static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb,
 733			     struct tipc_bearer *b,
 734			     struct tipc_media_addr *dst,
 735			     struct tipc_node *__dnode)
 736{
 737	struct crypto_aead *tfm = tipc_aead_tfm_next(aead);
 738	struct tipc_crypto_tx_ctx *tx_ctx;
 739	struct aead_request *req;
 740	struct sk_buff *trailer;
 741	struct scatterlist *sg;
 742	struct tipc_ehdr *ehdr;
 743	int ehsz, len, tailen, nsg, rc;
 744	void *ctx;
 745	u32 salt;
 746	u8 *iv;
 747
 748	/* Make sure message len at least 4-byte aligned */
 749	len = ALIGN(skb->len, 4);
 750	tailen = len - skb->len + aead->authsize;
 751
 752	/* Expand skb tail for authentication tag:
 753	 * As for simplicity, we'd have made sure skb having enough tailroom
 754	 * for authentication tag @skb allocation. Even when skb is nonlinear
 755	 * but there is no frag_list, it should be still fine!
 756	 * Otherwise, we must cow it to be a writable buffer with the tailroom.
 757	 */
 758	SKB_LINEAR_ASSERT(skb);
 759	if (tailen > skb_tailroom(skb)) {
 760		pr_debug("TX(): skb tailroom is not enough: %d, requires: %d\n",
 761			 skb_tailroom(skb), tailen);
 762	}
 763
 764	nsg = skb_cow_data(skb, tailen, &trailer);
 765	if (unlikely(nsg < 0)) {
 766		pr_err("TX: skb_cow_data() returned %d\n", nsg);
 767		return nsg;
 768	}
 769
 770	pskb_put(skb, trailer, tailen);
 771
 772	/* Allocate memory for the AEAD operation */
 773	ctx = tipc_aead_mem_alloc(tfm, sizeof(*tx_ctx), &iv, &req, &sg, nsg);
 774	if (unlikely(!ctx))
 775		return -ENOMEM;
 776	TIPC_SKB_CB(skb)->crypto_ctx = ctx;
 777
 778	/* Map skb to the sg lists */
 779	sg_init_table(sg, nsg);
 780	rc = skb_to_sgvec(skb, sg, 0, skb->len);
 781	if (unlikely(rc < 0)) {
 782		pr_err("TX: skb_to_sgvec() returned %d, nsg %d!\n", rc, nsg);
 783		goto exit;
 784	}
 785
 786	/* Prepare IV: [SALT (4 octets)][SEQNO (8 octets)]
 787	 * In case we're in cluster-key mode, SALT is varied by xor-ing with
 788	 * the source address (or w0 of id), otherwise with the dest address
 789	 * if dest is known.
 790	 */
 791	ehdr = (struct tipc_ehdr *)skb->data;
 792	salt = aead->salt;
 793	if (aead->mode == CLUSTER_KEY)
 794		salt ^= __be32_to_cpu(ehdr->addr);
 795	else if (__dnode)
 796		salt ^= tipc_node_get_addr(__dnode);
 797	memcpy(iv, &salt, 4);
 798	memcpy(iv + 4, (u8 *)&ehdr->seqno, 8);
 799
 800	/* Prepare request */
 801	ehsz = tipc_ehdr_size(ehdr);
 802	aead_request_set_tfm(req, tfm);
 803	aead_request_set_ad(req, ehsz);
 804	aead_request_set_crypt(req, sg, sg, len - ehsz, iv);
 805
 806	/* Set callback function & data */
 807	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 808				  tipc_aead_encrypt_done, skb);
 809	tx_ctx = (struct tipc_crypto_tx_ctx *)ctx;
 810	tx_ctx->aead = aead;
 811	tx_ctx->bearer = b;
 812	memcpy(&tx_ctx->dst, dst, sizeof(*dst));
 813
 814	/* Hold bearer */
 815	if (unlikely(!tipc_bearer_hold(b))) {
 816		rc = -ENODEV;
 817		goto exit;
 818	}
 819
 820	/* Get net to avoid freed tipc_crypto when delete namespace */
 821	if (!maybe_get_net(aead->crypto->net)) {
 822		tipc_bearer_put(b);
 823		rc = -ENODEV;
 824		goto exit;
 825	}
 826
 827	/* Now, do encrypt */
 828	rc = crypto_aead_encrypt(req);
 829	if (rc == -EINPROGRESS || rc == -EBUSY)
 830		return rc;
 831
 832	tipc_bearer_put(b);
 833	put_net(aead->crypto->net);
 834
 835exit:
 836	kfree(ctx);
 837	TIPC_SKB_CB(skb)->crypto_ctx = NULL;
 838	return rc;
 839}
 840
 841static void tipc_aead_encrypt_done(void *data, int err)
 842{
 843	struct sk_buff *skb = data;
 844	struct tipc_crypto_tx_ctx *tx_ctx = TIPC_SKB_CB(skb)->crypto_ctx;
 845	struct tipc_bearer *b = tx_ctx->bearer;
 846	struct tipc_aead *aead = tx_ctx->aead;
 847	struct tipc_crypto *tx = aead->crypto;
 848	struct net *net = tx->net;
 849
 850	switch (err) {
 851	case 0:
 852		this_cpu_inc(tx->stats->stat[STAT_ASYNC_OK]);
 853		rcu_read_lock();
 854		if (likely(test_bit(0, &b->up)))
 855			b->media->send_msg(net, skb, b, &tx_ctx->dst);
 856		else
 857			kfree_skb(skb);
 858		rcu_read_unlock();
 859		break;
 860	case -EINPROGRESS:
 861		return;
 862	default:
 863		this_cpu_inc(tx->stats->stat[STAT_ASYNC_NOK]);
 864		kfree_skb(skb);
 865		break;
 866	}
 867
 868	kfree(tx_ctx);
 869	tipc_bearer_put(b);
 870	tipc_aead_put(aead);
 871	put_net(net);
 872}
 873
 874/**
 875 * tipc_aead_decrypt - Decrypt an encrypted message
 876 * @net: struct net
 877 * @aead: TIPC AEAD for the message decryption
 878 * @skb: the input/output skb
 879 * @b: TIPC bearer where the message has been received
 880 *
 881 * Return:
 882 * * 0                   : if the decryption has completed
 883 * * -EINPROGRESS/-EBUSY : if a callback will be performed
 884 * * < 0                 : the decryption has failed
 885 */
 886static int tipc_aead_decrypt(struct net *net, struct tipc_aead *aead,
 887			     struct sk_buff *skb, struct tipc_bearer *b)
 888{
 889	struct tipc_crypto_rx_ctx *rx_ctx;
 890	struct aead_request *req;
 891	struct crypto_aead *tfm;
 892	struct sk_buff *unused;
 893	struct scatterlist *sg;
 894	struct tipc_ehdr *ehdr;
 895	int ehsz, nsg, rc;
 896	void *ctx;
 897	u32 salt;
 898	u8 *iv;
 899
 900	if (unlikely(!aead))
 901		return -ENOKEY;
 902
 903	nsg = skb_cow_data(skb, 0, &unused);
 904	if (unlikely(nsg < 0)) {
 905		pr_err("RX: skb_cow_data() returned %d\n", nsg);
 906		return nsg;
 907	}
 908
 909	/* Allocate memory for the AEAD operation */
 910	tfm = tipc_aead_tfm_next(aead);
 911	ctx = tipc_aead_mem_alloc(tfm, sizeof(*rx_ctx), &iv, &req, &sg, nsg);
 912	if (unlikely(!ctx))
 913		return -ENOMEM;
 914	TIPC_SKB_CB(skb)->crypto_ctx = ctx;
 915
 916	/* Map skb to the sg lists */
 917	sg_init_table(sg, nsg);
 918	rc = skb_to_sgvec(skb, sg, 0, skb->len);
 919	if (unlikely(rc < 0)) {
 920		pr_err("RX: skb_to_sgvec() returned %d, nsg %d\n", rc, nsg);
 921		goto exit;
 922	}
 923
 924	/* Reconstruct IV: */
 925	ehdr = (struct tipc_ehdr *)skb->data;
 926	salt = aead->salt;
 927	if (aead->mode == CLUSTER_KEY)
 928		salt ^= __be32_to_cpu(ehdr->addr);
 929	else if (ehdr->destined)
 930		salt ^= tipc_own_addr(net);
 931	memcpy(iv, &salt, 4);
 932	memcpy(iv + 4, (u8 *)&ehdr->seqno, 8);
 933
 934	/* Prepare request */
 935	ehsz = tipc_ehdr_size(ehdr);
 936	aead_request_set_tfm(req, tfm);
 937	aead_request_set_ad(req, ehsz);
 938	aead_request_set_crypt(req, sg, sg, skb->len - ehsz, iv);
 939
 940	/* Set callback function & data */
 941	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 942				  tipc_aead_decrypt_done, skb);
 943	rx_ctx = (struct tipc_crypto_rx_ctx *)ctx;
 944	rx_ctx->aead = aead;
 945	rx_ctx->bearer = b;
 946
 947	/* Hold bearer */
 948	if (unlikely(!tipc_bearer_hold(b))) {
 949		rc = -ENODEV;
 950		goto exit;
 951	}
 952
 953	/* Now, do decrypt */
 954	rc = crypto_aead_decrypt(req);
 955	if (rc == -EINPROGRESS || rc == -EBUSY)
 956		return rc;
 957
 958	tipc_bearer_put(b);
 959
 960exit:
 961	kfree(ctx);
 962	TIPC_SKB_CB(skb)->crypto_ctx = NULL;
 963	return rc;
 964}
 965
 966static void tipc_aead_decrypt_done(void *data, int err)
 967{
 968	struct sk_buff *skb = data;
 969	struct tipc_crypto_rx_ctx *rx_ctx = TIPC_SKB_CB(skb)->crypto_ctx;
 970	struct tipc_bearer *b = rx_ctx->bearer;
 971	struct tipc_aead *aead = rx_ctx->aead;
 972	struct tipc_crypto_stats __percpu *stats = aead->crypto->stats;
 973	struct net *net = aead->crypto->net;
 974
 975	switch (err) {
 976	case 0:
 977		this_cpu_inc(stats->stat[STAT_ASYNC_OK]);
 978		break;
 979	case -EINPROGRESS:
 980		return;
 981	default:
 982		this_cpu_inc(stats->stat[STAT_ASYNC_NOK]);
 983		break;
 984	}
 985
 986	kfree(rx_ctx);
 987	tipc_crypto_rcv_complete(net, aead, b, &skb, err);
 988	if (likely(skb)) {
 989		if (likely(test_bit(0, &b->up)))
 990			tipc_rcv(net, skb, b);
 991		else
 992			kfree_skb(skb);
 993	}
 994
 995	tipc_bearer_put(b);
 996}
 997
 998static inline int tipc_ehdr_size(struct tipc_ehdr *ehdr)
 999{
1000	return (ehdr->user != LINK_CONFIG) ? EHDR_SIZE : EHDR_CFG_SIZE;
1001}
1002
1003/**
1004 * tipc_ehdr_validate - Validate an encryption message
1005 * @skb: the message buffer
1006 *
1007 * Return: "true" if this is a valid encryption message, otherwise "false"
1008 */
1009bool tipc_ehdr_validate(struct sk_buff *skb)
1010{
1011	struct tipc_ehdr *ehdr;
1012	int ehsz;
1013
1014	if (unlikely(!pskb_may_pull(skb, EHDR_MIN_SIZE)))
1015		return false;
1016
1017	ehdr = (struct tipc_ehdr *)skb->data;
1018	if (unlikely(ehdr->version != TIPC_EVERSION))
1019		return false;
1020	ehsz = tipc_ehdr_size(ehdr);
1021	if (unlikely(!pskb_may_pull(skb, ehsz)))
1022		return false;
1023	if (unlikely(skb->len <= ehsz + TIPC_AES_GCM_TAG_SIZE))
1024		return false;
1025
1026	return true;
1027}
1028
1029/**
1030 * tipc_ehdr_build - Build TIPC encryption message header
1031 * @net: struct net
1032 * @aead: TX AEAD key to be used for the message encryption
1033 * @tx_key: key id used for the message encryption
1034 * @skb: input/output message skb
1035 * @__rx: RX crypto handle if dest is "known"
1036 *
1037 * Return: the header size if the building is successful, otherwise < 0
1038 */
1039static int tipc_ehdr_build(struct net *net, struct tipc_aead *aead,
1040			   u8 tx_key, struct sk_buff *skb,
1041			   struct tipc_crypto *__rx)
1042{
1043	struct tipc_msg *hdr = buf_msg(skb);
1044	struct tipc_ehdr *ehdr;
1045	u32 user = msg_user(hdr);
1046	u64 seqno;
1047	int ehsz;
1048
1049	/* Make room for encryption header */
1050	ehsz = (user != LINK_CONFIG) ? EHDR_SIZE : EHDR_CFG_SIZE;
1051	WARN_ON(skb_headroom(skb) < ehsz);
1052	ehdr = (struct tipc_ehdr *)skb_push(skb, ehsz);
1053
1054	/* Obtain a seqno first:
1055	 * Use the key seqno (= cluster wise) if dest is unknown or we're in
1056	 * cluster key mode, otherwise it's better for a per-peer seqno!
1057	 */
1058	if (!__rx || aead->mode == CLUSTER_KEY)
1059		seqno = atomic64_inc_return(&aead->seqno);
1060	else
1061		seqno = atomic64_inc_return(&__rx->sndnxt);
1062
1063	/* Revoke the key if seqno is wrapped around */
1064	if (unlikely(!seqno))
1065		return tipc_crypto_key_revoke(net, tx_key);
1066
1067	/* Word 1-2 */
1068	ehdr->seqno = cpu_to_be64(seqno);
1069
1070	/* Words 0, 3- */
1071	ehdr->version = TIPC_EVERSION;
1072	ehdr->user = 0;
1073	ehdr->keepalive = 0;
1074	ehdr->tx_key = tx_key;
1075	ehdr->destined = (__rx) ? 1 : 0;
1076	ehdr->rx_key_active = (__rx) ? __rx->key.active : 0;
1077	ehdr->rx_nokey = (__rx) ? __rx->nokey : 0;
1078	ehdr->master_key = aead->crypto->key_master;
1079	ehdr->reserved_1 = 0;
1080	ehdr->reserved_2 = 0;
1081
1082	switch (user) {
1083	case LINK_CONFIG:
1084		ehdr->user = LINK_CONFIG;
1085		memcpy(ehdr->id, tipc_own_id(net), NODE_ID_LEN);
1086		break;
1087	default:
1088		if (user == LINK_PROTOCOL && msg_type(hdr) == STATE_MSG) {
1089			ehdr->user = LINK_PROTOCOL;
1090			ehdr->keepalive = msg_is_keepalive(hdr);
1091		}
1092		ehdr->addr = hdr->hdr[3];
1093		break;
1094	}
1095
1096	return ehsz;
1097}
1098
1099static inline void tipc_crypto_key_set_state(struct tipc_crypto *c,
1100					     u8 new_passive,
1101					     u8 new_active,
1102					     u8 new_pending)
1103{
1104	struct tipc_key old = c->key;
1105	char buf[32];
1106
1107	c->key.keys = ((new_passive & KEY_MASK) << (KEY_BITS * 2)) |
1108		      ((new_active  & KEY_MASK) << (KEY_BITS)) |
1109		      ((new_pending & KEY_MASK));
1110
1111	pr_debug("%s: key changing %s ::%pS\n", c->name,
1112		 tipc_key_change_dump(old, c->key, buf),
1113		 __builtin_return_address(0));
1114}
1115
1116/**
1117 * tipc_crypto_key_init - Initiate a new user / AEAD key
1118 * @c: TIPC crypto to which new key is attached
1119 * @ukey: the user key
1120 * @mode: the key mode (CLUSTER_KEY or PER_NODE_KEY)
1121 * @master_key: specify this is a cluster master key
1122 *
1123 * A new TIPC AEAD key will be allocated and initiated with the specified user
1124 * key, then attached to the TIPC crypto.
1125 *
1126 * Return: new key id in case of success, otherwise: < 0
1127 */
1128int tipc_crypto_key_init(struct tipc_crypto *c, struct tipc_aead_key *ukey,
1129			 u8 mode, bool master_key)
1130{
1131	struct tipc_aead *aead = NULL;
1132	int rc = 0;
1133
1134	/* Initiate with the new user key */
1135	rc = tipc_aead_init(&aead, ukey, mode);
1136
1137	/* Attach it to the crypto */
1138	if (likely(!rc)) {
1139		rc = tipc_crypto_key_attach(c, aead, 0, master_key);
1140		if (rc < 0)
1141			tipc_aead_free(&aead->rcu);
1142	}
1143
1144	return rc;
1145}
1146
1147/**
1148 * tipc_crypto_key_attach - Attach a new AEAD key to TIPC crypto
1149 * @c: TIPC crypto to which the new AEAD key is attached
1150 * @aead: the new AEAD key pointer
1151 * @pos: desired slot in the crypto key array, = 0 if any!
1152 * @master_key: specify this is a cluster master key
1153 *
1154 * Return: new key id in case of success, otherwise: -EBUSY
1155 */
1156static int tipc_crypto_key_attach(struct tipc_crypto *c,
1157				  struct tipc_aead *aead, u8 pos,
1158				  bool master_key)
1159{
1160	struct tipc_key key;
1161	int rc = -EBUSY;
1162	u8 new_key;
1163
1164	spin_lock_bh(&c->lock);
1165	key = c->key;
1166	if (master_key) {
1167		new_key = KEY_MASTER;
1168		goto attach;
1169	}
1170	if (key.active && key.passive)
1171		goto exit;
1172	if (key.pending) {
1173		if (tipc_aead_users(c->aead[key.pending]) > 0)
1174			goto exit;
1175		/* if (pos): ok with replacing, will be aligned when needed */
1176		/* Replace it */
1177		new_key = key.pending;
1178	} else {
1179		if (pos) {
1180			if (key.active && pos != key_next(key.active)) {
1181				key.passive = pos;
1182				new_key = pos;
1183				goto attach;
1184			} else if (!key.active && !key.passive) {
1185				key.pending = pos;
1186				new_key = pos;
1187				goto attach;
1188			}
1189		}
1190		key.pending = key_next(key.active ?: key.passive);
1191		new_key = key.pending;
1192	}
1193
1194attach:
1195	aead->crypto = c;
1196	aead->gen = (is_tx(c)) ? ++c->key_gen : c->key_gen;
1197	tipc_aead_rcu_replace(c->aead[new_key], aead, &c->lock);
1198	if (likely(c->key.keys != key.keys))
1199		tipc_crypto_key_set_state(c, key.passive, key.active,
1200					  key.pending);
1201	c->working = 1;
1202	c->nokey = 0;
1203	c->key_master |= master_key;
1204	rc = new_key;
1205
1206exit:
1207	spin_unlock_bh(&c->lock);
1208	return rc;
1209}
1210
1211void tipc_crypto_key_flush(struct tipc_crypto *c)
1212{
1213	struct tipc_crypto *tx, *rx;
1214	int k;
1215
1216	spin_lock_bh(&c->lock);
1217	if (is_rx(c)) {
1218		/* Try to cancel pending work */
1219		rx = c;
1220		tx = tipc_net(rx->net)->crypto_tx;
1221		if (cancel_delayed_work(&rx->work)) {
1222			kfree_sensitive(rx->skey);
1223			rx->skey = NULL;
1224			atomic_xchg(&rx->key_distr, 0);
1225			tipc_node_put(rx->node);
1226		}
1227		/* RX stopping => decrease TX key users if any */
1228		k = atomic_xchg(&rx->peer_rx_active, 0);
1229		if (k) {
1230			tipc_aead_users_dec(tx->aead[k], 0);
1231			/* Mark the point TX key users changed */
1232			tx->timer1 = jiffies;
1233		}
1234	}
1235
1236	c->flags = 0;
1237	tipc_crypto_key_set_state(c, 0, 0, 0);
1238	for (k = KEY_MIN; k <= KEY_MAX; k++)
1239		tipc_crypto_key_detach(c->aead[k], &c->lock);
1240	atomic64_set(&c->sndnxt, 0);
1241	spin_unlock_bh(&c->lock);
1242}
1243
1244/**
1245 * tipc_crypto_key_try_align - Align RX keys if possible
1246 * @rx: RX crypto handle
1247 * @new_pending: new pending slot if aligned (= TX key from peer)
1248 *
1249 * Peer has used an unknown key slot, this only happens when peer has left and
1250 * rejoned, or we are newcomer.
1251 * That means, there must be no active key but a pending key at unaligned slot.
1252 * If so, we try to move the pending key to the new slot.
1253 * Note: A potential passive key can exist, it will be shifted correspondingly!
1254 *
1255 * Return: "true" if key is successfully aligned, otherwise "false"
1256 */
1257static bool tipc_crypto_key_try_align(struct tipc_crypto *rx, u8 new_pending)
1258{
1259	struct tipc_aead *tmp1, *tmp2 = NULL;
1260	struct tipc_key key;
1261	bool aligned = false;
1262	u8 new_passive = 0;
1263	int x;
1264
1265	spin_lock(&rx->lock);
1266	key = rx->key;
1267	if (key.pending == new_pending) {
1268		aligned = true;
1269		goto exit;
1270	}
1271	if (key.active)
1272		goto exit;
1273	if (!key.pending)
1274		goto exit;
1275	if (tipc_aead_users(rx->aead[key.pending]) > 0)
1276		goto exit;
1277
1278	/* Try to "isolate" this pending key first */
1279	tmp1 = tipc_aead_rcu_ptr(rx->aead[key.pending], &rx->lock);
1280	if (!refcount_dec_if_one(&tmp1->refcnt))
1281		goto exit;
1282	rcu_assign_pointer(rx->aead[key.pending], NULL);
1283
1284	/* Move passive key if any */
1285	if (key.passive) {
1286		tmp2 = rcu_replace_pointer(rx->aead[key.passive], tmp2, lockdep_is_held(&rx->lock));
1287		x = (key.passive - key.pending + new_pending) % KEY_MAX;
1288		new_passive = (x <= 0) ? x + KEY_MAX : x;
1289	}
1290
1291	/* Re-allocate the key(s) */
1292	tipc_crypto_key_set_state(rx, new_passive, 0, new_pending);
1293	rcu_assign_pointer(rx->aead[new_pending], tmp1);
1294	if (new_passive)
1295		rcu_assign_pointer(rx->aead[new_passive], tmp2);
1296	refcount_set(&tmp1->refcnt, 1);
1297	aligned = true;
1298	pr_info_ratelimited("%s: key[%d] -> key[%d]\n", rx->name, key.pending,
1299			    new_pending);
1300
1301exit:
1302	spin_unlock(&rx->lock);
1303	return aligned;
1304}
1305
1306/**
1307 * tipc_crypto_key_pick_tx - Pick one TX key for message decryption
1308 * @tx: TX crypto handle
1309 * @rx: RX crypto handle (can be NULL)
1310 * @skb: the message skb which will be decrypted later
1311 * @tx_key: peer TX key id
1312 *
1313 * This function looks up the existing TX keys and pick one which is suitable
1314 * for the message decryption, that must be a cluster key and not used before
1315 * on the same message (i.e. recursive).
1316 *
1317 * Return: the TX AEAD key handle in case of success, otherwise NULL
1318 */
1319static struct tipc_aead *tipc_crypto_key_pick_tx(struct tipc_crypto *tx,
1320						 struct tipc_crypto *rx,
1321						 struct sk_buff *skb,
1322						 u8 tx_key)
1323{
1324	struct tipc_skb_cb *skb_cb = TIPC_SKB_CB(skb);
1325	struct tipc_aead *aead = NULL;
1326	struct tipc_key key = tx->key;
1327	u8 k, i = 0;
1328
1329	/* Initialize data if not yet */
1330	if (!skb_cb->tx_clone_deferred) {
1331		skb_cb->tx_clone_deferred = 1;
1332		memset(&skb_cb->tx_clone_ctx, 0, sizeof(skb_cb->tx_clone_ctx));
1333	}
1334
1335	skb_cb->tx_clone_ctx.rx = rx;
1336	if (++skb_cb->tx_clone_ctx.recurs > 2)
1337		return NULL;
1338
1339	/* Pick one TX key */
1340	spin_lock(&tx->lock);
1341	if (tx_key == KEY_MASTER) {
1342		aead = tipc_aead_rcu_ptr(tx->aead[KEY_MASTER], &tx->lock);
1343		goto done;
1344	}
1345	do {
1346		k = (i == 0) ? key.pending :
1347			((i == 1) ? key.active : key.passive);
1348		if (!k)
1349			continue;
1350		aead = tipc_aead_rcu_ptr(tx->aead[k], &tx->lock);
1351		if (!aead)
1352			continue;
1353		if (aead->mode != CLUSTER_KEY ||
1354		    aead == skb_cb->tx_clone_ctx.last) {
1355			aead = NULL;
1356			continue;
1357		}
1358		/* Ok, found one cluster key */
1359		skb_cb->tx_clone_ctx.last = aead;
1360		WARN_ON(skb->next);
1361		skb->next = skb_clone(skb, GFP_ATOMIC);
1362		if (unlikely(!skb->next))
1363			pr_warn("Failed to clone skb for next round if any\n");
1364		break;
1365	} while (++i < 3);
1366
1367done:
1368	if (likely(aead))
1369		WARN_ON(!refcount_inc_not_zero(&aead->refcnt));
1370	spin_unlock(&tx->lock);
1371
1372	return aead;
1373}
1374
1375/**
1376 * tipc_crypto_key_synch: Synch own key data according to peer key status
1377 * @rx: RX crypto handle
1378 * @skb: TIPCv2 message buffer (incl. the ehdr from peer)
1379 *
1380 * This function updates the peer node related data as the peer RX active key
1381 * has changed, so the number of TX keys' users on this node are increased and
1382 * decreased correspondingly.
1383 *
1384 * It also considers if peer has no key, then we need to make own master key
1385 * (if any) taking over i.e. starting grace period and also trigger key
1386 * distributing process.
1387 *
1388 * The "per-peer" sndnxt is also reset when the peer key has switched.
1389 */
1390static void tipc_crypto_key_synch(struct tipc_crypto *rx, struct sk_buff *skb)
1391{
1392	struct tipc_ehdr *ehdr = (struct tipc_ehdr *)skb_network_header(skb);
1393	struct tipc_crypto *tx = tipc_net(rx->net)->crypto_tx;
1394	struct tipc_msg *hdr = buf_msg(skb);
1395	u32 self = tipc_own_addr(rx->net);
1396	u8 cur, new;
1397	unsigned long delay;
1398
1399	/* Update RX 'key_master' flag according to peer, also mark "legacy" if
1400	 * a peer has no master key.
1401	 */
1402	rx->key_master = ehdr->master_key;
1403	if (!rx->key_master)
1404		tx->legacy_user = 1;
1405
1406	/* For later cases, apply only if message is destined to this node */
1407	if (!ehdr->destined || msg_short(hdr) || msg_destnode(hdr) != self)
1408		return;
1409
1410	/* Case 1: Peer has no keys, let's make master key take over */
1411	if (ehdr->rx_nokey) {
1412		/* Set or extend grace period */
1413		tx->timer2 = jiffies;
1414		/* Schedule key distributing for the peer if not yet */
1415		if (tx->key.keys &&
1416		    !atomic_cmpxchg(&rx->key_distr, 0, KEY_DISTR_SCHED)) {
1417			get_random_bytes(&delay, 2);
1418			delay %= 5;
1419			delay = msecs_to_jiffies(500 * ++delay);
1420			if (queue_delayed_work(tx->wq, &rx->work, delay))
1421				tipc_node_get(rx->node);
1422		}
1423	} else {
1424		/* Cancel a pending key distributing if any */
1425		atomic_xchg(&rx->key_distr, 0);
1426	}
1427
1428	/* Case 2: Peer RX active key has changed, let's update own TX users */
1429	cur = atomic_read(&rx->peer_rx_active);
1430	new = ehdr->rx_key_active;
1431	if (tx->key.keys &&
1432	    cur != new &&
1433	    atomic_cmpxchg(&rx->peer_rx_active, cur, new) == cur) {
1434		if (new)
1435			tipc_aead_users_inc(tx->aead[new], INT_MAX);
1436		if (cur)
1437			tipc_aead_users_dec(tx->aead[cur], 0);
1438
1439		atomic64_set(&rx->sndnxt, 0);
1440		/* Mark the point TX key users changed */
1441		tx->timer1 = jiffies;
1442
1443		pr_debug("%s: key users changed %d-- %d++, peer %s\n",
1444			 tx->name, cur, new, rx->name);
1445	}
1446}
1447
1448static int tipc_crypto_key_revoke(struct net *net, u8 tx_key)
1449{
1450	struct tipc_crypto *tx = tipc_net(net)->crypto_tx;
1451	struct tipc_key key;
1452
1453	spin_lock_bh(&tx->lock);
1454	key = tx->key;
1455	WARN_ON(!key.active || tx_key != key.active);
1456
1457	/* Free the active key */
1458	tipc_crypto_key_set_state(tx, key.passive, 0, key.pending);
1459	tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);
1460	spin_unlock_bh(&tx->lock);
1461
1462	pr_warn("%s: key is revoked\n", tx->name);
1463	return -EKEYREVOKED;
1464}
1465
1466int tipc_crypto_start(struct tipc_crypto **crypto, struct net *net,
1467		      struct tipc_node *node)
1468{
1469	struct tipc_crypto *c;
1470
1471	if (*crypto)
1472		return -EEXIST;
1473
1474	/* Allocate crypto */
1475	c = kzalloc(sizeof(*c), GFP_ATOMIC);
1476	if (!c)
1477		return -ENOMEM;
1478
1479	/* Allocate workqueue on TX */
1480	if (!node) {
1481		c->wq = alloc_ordered_workqueue("tipc_crypto", 0);
1482		if (!c->wq) {
1483			kfree(c);
1484			return -ENOMEM;
1485		}
1486	}
1487
1488	/* Allocate statistic structure */
1489	c->stats = alloc_percpu_gfp(struct tipc_crypto_stats, GFP_ATOMIC);
1490	if (!c->stats) {
1491		if (c->wq)
1492			destroy_workqueue(c->wq);
1493		kfree_sensitive(c);
1494		return -ENOMEM;
1495	}
1496
1497	c->flags = 0;
1498	c->net = net;
1499	c->node = node;
1500	get_random_bytes(&c->key_gen, 2);
1501	tipc_crypto_key_set_state(c, 0, 0, 0);
1502	atomic_set(&c->key_distr, 0);
1503	atomic_set(&c->peer_rx_active, 0);
1504	atomic64_set(&c->sndnxt, 0);
1505	c->timer1 = jiffies;
1506	c->timer2 = jiffies;
1507	c->rekeying_intv = TIPC_REKEYING_INTV_DEF;
1508	spin_lock_init(&c->lock);
1509	scnprintf(c->name, 48, "%s(%s)", (is_rx(c)) ? "RX" : "TX",
1510		  (is_rx(c)) ? tipc_node_get_id_str(c->node) :
1511			       tipc_own_id_string(c->net));
1512
1513	if (is_rx(c))
1514		INIT_DELAYED_WORK(&c->work, tipc_crypto_work_rx);
1515	else
1516		INIT_DELAYED_WORK(&c->work, tipc_crypto_work_tx);
1517
1518	*crypto = c;
1519	return 0;
1520}
1521
1522void tipc_crypto_stop(struct tipc_crypto **crypto)
1523{
1524	struct tipc_crypto *c = *crypto;
1525	u8 k;
1526
1527	if (!c)
1528		return;
1529
1530	/* Flush any queued works & destroy wq */
1531	if (is_tx(c)) {
1532		c->rekeying_intv = 0;
1533		cancel_delayed_work_sync(&c->work);
1534		destroy_workqueue(c->wq);
1535	}
1536
1537	/* Release AEAD keys */
1538	rcu_read_lock();
1539	for (k = KEY_MIN; k <= KEY_MAX; k++)
1540		tipc_aead_put(rcu_dereference(c->aead[k]));
1541	rcu_read_unlock();
1542	pr_debug("%s: has been stopped\n", c->name);
1543
1544	/* Free this crypto statistics */
1545	free_percpu(c->stats);
1546
1547	*crypto = NULL;
1548	kfree_sensitive(c);
1549}
1550
1551void tipc_crypto_timeout(struct tipc_crypto *rx)
1552{
1553	struct tipc_net *tn = tipc_net(rx->net);
1554	struct tipc_crypto *tx = tn->crypto_tx;
1555	struct tipc_key key;
1556	int cmd;
1557
1558	/* TX pending: taking all users & stable -> active */
1559	spin_lock(&tx->lock);
1560	key = tx->key;
1561	if (key.active && tipc_aead_users(tx->aead[key.active]) > 0)
1562		goto s1;
1563	if (!key.pending || tipc_aead_users(tx->aead[key.pending]) <= 0)
1564		goto s1;
1565	if (time_before(jiffies, tx->timer1 + TIPC_TX_LASTING_TIME))
1566		goto s1;
1567
1568	tipc_crypto_key_set_state(tx, key.passive, key.pending, 0);
1569	if (key.active)
1570		tipc_crypto_key_detach(tx->aead[key.active], &tx->lock);
1571	this_cpu_inc(tx->stats->stat[STAT_SWITCHES]);
1572	pr_info("%s: key[%d] is activated\n", tx->name, key.pending);
1573
1574s1:
1575	spin_unlock(&tx->lock);
1576
1577	/* RX pending: having user -> active */
1578	spin_lock(&rx->lock);
1579	key = rx->key;
1580	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) <= 0)
1581		goto s2;
1582
1583	if (key.active)
1584		key.passive = key.active;
1585	key.active = key.pending;
1586	rx->timer2 = jiffies;
1587	tipc_crypto_key_set_state(rx, key.passive, key.active, 0);
1588	this_cpu_inc(rx->stats->stat[STAT_SWITCHES]);
1589	pr_info("%s: key[%d] is activated\n", rx->name, key.pending);
1590	goto s5;
1591
1592s2:
1593	/* RX pending: not working -> remove */
1594	if (!key.pending || tipc_aead_users(rx->aead[key.pending]) > -10)
1595		goto s3;
1596
1597	tipc_crypto_key_set_state(rx, key.passive, key.active, 0);
1598	tipc_crypto_key_detach(rx->aead[key.pending], &rx->lock);
1599	pr_debug("%s: key[%d] is removed\n", rx->name, key.pending);
1600	goto s5;
1601
1602s3:
1603	/* RX active: timed out or no user -> pending */
1604	if (!key.active)
1605		goto s4;
1606	if (time_before(jiffies, rx->timer1 + TIPC_RX_ACTIVE_LIM) &&
1607	    tipc_aead_users(rx->aead[key.active]) > 0)
1608		goto s4;
1609
1610	if (key.pending)
1611		key.passive = key.active;
1612	else
1613		key.pending = key.active;
1614	rx->timer2 = jiffies;
1615	tipc_crypto_key_set_state(rx, key.passive, 0, key.pending);
1616	tipc_aead_users_set(rx->aead[key.pending], 0);
1617	pr_debug("%s: key[%d] is deactivated\n", rx->name, key.active);
1618	goto s5;
1619
1620s4:
1621	/* RX passive: outdated or not working -> free */
1622	if (!key.passive)
1623		goto s5;
1624	if (time_before(jiffies, rx->timer2 + TIPC_RX_PASSIVE_LIM) &&
1625	    tipc_aead_users(rx->aead[key.passive]) > -10)
1626		goto s5;
1627
1628	tipc_crypto_key_set_state(rx, 0, key.active, key.pending);
1629	tipc_crypto_key_detach(rx->aead[key.passive], &rx->lock);
1630	pr_debug("%s: key[%d] is freed\n", rx->name, key.passive);
1631
1632s5:
1633	spin_unlock(&rx->lock);
1634
1635	/* Relax it here, the flag will be set again if it really is, but only
1636	 * when we are not in grace period for safety!
1637	 */
1638	if (time_after(jiffies, tx->timer2 + TIPC_TX_GRACE_PERIOD))
1639		tx->legacy_user = 0;
1640
1641	/* Limit max_tfms & do debug commands if needed */
1642	if (likely(sysctl_tipc_max_tfms <= TIPC_MAX_TFMS_LIM))
1643		return;
1644
1645	cmd = sysctl_tipc_max_tfms;
1646	sysctl_tipc_max_tfms = TIPC_MAX_TFMS_DEF;
1647	tipc_crypto_do_cmd(rx->net, cmd);
1648}
1649
1650static inline void tipc_crypto_clone_msg(struct net *net, struct sk_buff *_skb,
1651					 struct tipc_bearer *b,
1652					 struct tipc_media_addr *dst,
1653					 struct tipc_node *__dnode, u8 type)
1654{
1655	struct sk_buff *skb;
1656
1657	skb = skb_clone(_skb, GFP_ATOMIC);
1658	if (skb) {
1659		TIPC_SKB_CB(skb)->xmit_type = type;
1660		tipc_crypto_xmit(net, &skb, b, dst, __dnode);
1661		if (skb)
1662			b->media->send_msg(net, skb, b, dst);
1663	}
1664}
1665
1666/**
1667 * tipc_crypto_xmit - Build & encrypt TIPC message for xmit
1668 * @net: struct net
1669 * @skb: input/output message skb pointer
1670 * @b: bearer used for xmit later
1671 * @dst: destination media address
1672 * @__dnode: destination node for reference if any
1673 *
1674 * First, build an encryption message header on the top of the message, then
1675 * encrypt the original TIPC message by using the pending, master or active
1676 * key with this preference order.
1677 * If the encryption is successful, the encrypted skb is returned directly or
1678 * via the callback.
1679 * Otherwise, the skb is freed!
1680 *
1681 * Return:
1682 * * 0                   : the encryption has succeeded (or no encryption)
1683 * * -EINPROGRESS/-EBUSY : the encryption is ongoing, a callback will be made
1684 * * -ENOKEK             : the encryption has failed due to no key
1685 * * -EKEYREVOKED        : the encryption has failed due to key revoked
1686 * * -ENOMEM             : the encryption has failed due to no memory
1687 * * < 0                 : the encryption has failed due to other reasons
1688 */
1689int tipc_crypto_xmit(struct net *net, struct sk_buff **skb,
1690		     struct tipc_bearer *b, struct tipc_media_addr *dst,
1691		     struct tipc_node *__dnode)
1692{
1693	struct tipc_crypto *__rx = tipc_node_crypto_rx(__dnode);
1694	struct tipc_crypto *tx = tipc_net(net)->crypto_tx;
1695	struct tipc_crypto_stats __percpu *stats = tx->stats;
1696	struct tipc_msg *hdr = buf_msg(*skb);
1697	struct tipc_key key = tx->key;
1698	struct tipc_aead *aead = NULL;
1699	u32 user = msg_user(hdr);
1700	u32 type = msg_type(hdr);
1701	int rc = -ENOKEY;
1702	u8 tx_key = 0;
1703
1704	/* No encryption? */
1705	if (!tx->working)
1706		return 0;
1707
1708	/* Pending key if peer has active on it or probing time */
1709	if (unlikely(key.pending)) {
1710		tx_key = key.pending;
1711		if (!tx->key_master && !key.active)
1712			goto encrypt;
1713		if (__rx && atomic_read(&__rx->peer_rx_active) == tx_key)
1714			goto encrypt;
1715		if (TIPC_SKB_CB(*skb)->xmit_type == SKB_PROBING) {
1716			pr_debug("%s: probing for key[%d]\n", tx->name,
1717				 key.pending);
1718			goto encrypt;
1719		}
1720		if (user == LINK_CONFIG || user == LINK_PROTOCOL)
1721			tipc_crypto_clone_msg(net, *skb, b, dst, __dnode,
1722					      SKB_PROBING);
1723	}
1724
1725	/* Master key if this is a *vital* message or in grace period */
1726	if (tx->key_master) {
1727		tx_key = KEY_MASTER;
1728		if (!key.active)
1729			goto encrypt;
1730		if (TIPC_SKB_CB(*skb)->xmit_type == SKB_GRACING) {
1731			pr_debug("%s: gracing for msg (%d %d)\n", tx->name,
1732				 user, type);
1733			goto encrypt;
1734		}
1735		if (user == LINK_CONFIG ||
1736		    (user == LINK_PROTOCOL && type == RESET_MSG) ||
1737		    (user == MSG_CRYPTO && type == KEY_DISTR_MSG) ||
1738		    time_before(jiffies, tx->timer2 + TIPC_TX_GRACE_PERIOD)) {
1739			if (__rx && __rx->key_master &&
1740			    !atomic_read(&__rx->peer_rx_active))
1741				goto encrypt;
1742			if (!__rx) {
1743				if (likely(!tx->legacy_user))
1744					goto encrypt;
1745				tipc_crypto_clone_msg(net, *skb, b, dst,
1746						      __dnode, SKB_GRACING);
1747			}
1748		}
1749	}
1750
1751	/* Else, use the active key if any */
1752	if (likely(key.active)) {
1753		tx_key = key.active;
1754		goto encrypt;
1755	}
1756
1757	goto exit;
1758
1759encrypt:
1760	aead = tipc_aead_get(tx->aead[tx_key]);
1761	if (unlikely(!aead))
1762		goto exit;
1763	rc = tipc_ehdr_build(net, aead, tx_key, *skb, __rx);
1764	if (likely(rc > 0))
1765		rc = tipc_aead_encrypt(aead, *skb, b, dst, __dnode);
1766
1767exit:
1768	switch (rc) {
1769	case 0:
1770		this_cpu_inc(stats->stat[STAT_OK]);
1771		break;
1772	case -EINPROGRESS:
1773	case -EBUSY:
1774		this_cpu_inc(stats->stat[STAT_ASYNC]);
1775		*skb = NULL;
1776		return rc;
1777	default:
1778		this_cpu_inc(stats->stat[STAT_NOK]);
1779		if (rc == -ENOKEY)
1780			this_cpu_inc(stats->stat[STAT_NOKEYS]);
1781		else if (rc == -EKEYREVOKED)
1782			this_cpu_inc(stats->stat[STAT_BADKEYS]);
1783		kfree_skb(*skb);
1784		*skb = NULL;
1785		break;
1786	}
1787
1788	tipc_aead_put(aead);
1789	return rc;
1790}
1791
1792/**
1793 * tipc_crypto_rcv - Decrypt an encrypted TIPC message from peer
1794 * @net: struct net
1795 * @rx: RX crypto handle
1796 * @skb: input/output message skb pointer
1797 * @b: bearer where the message has been received
1798 *
1799 * If the decryption is successful, the decrypted skb is returned directly or
1800 * as the callback, the encryption header and auth tag will be trimmed out
1801 * before forwarding to tipc_rcv() via the tipc_crypto_rcv_complete().
1802 * Otherwise, the skb will be freed!
1803 * Note: RX key(s) can be re-aligned, or in case of no key suitable, TX
1804 * cluster key(s) can be taken for decryption (- recursive).
1805 *
1806 * Return:
1807 * * 0                   : the decryption has successfully completed
1808 * * -EINPROGRESS/-EBUSY : the decryption is ongoing, a callback will be made
1809 * * -ENOKEY             : the decryption has failed due to no key
1810 * * -EBADMSG            : the decryption has failed due to bad message
1811 * * -ENOMEM             : the decryption has failed due to no memory
1812 * * < 0                 : the decryption has failed due to other reasons
1813 */
1814int tipc_crypto_rcv(struct net *net, struct tipc_crypto *rx,
1815		    struct sk_buff **skb, struct tipc_bearer *b)
1816{
1817	struct tipc_crypto *tx = tipc_net(net)->crypto_tx;
1818	struct tipc_crypto_stats __percpu *stats;
1819	struct tipc_aead *aead = NULL;
1820	struct tipc_key key;
1821	int rc = -ENOKEY;
1822	u8 tx_key, n;
1823
1824	tx_key = ((struct tipc_ehdr *)(*skb)->data)->tx_key;
1825
1826	/* New peer?
1827	 * Let's try with TX key (i.e. cluster mode) & verify the skb first!
1828	 */
1829	if (unlikely(!rx || tx_key == KEY_MASTER))
1830		goto pick_tx;
1831
1832	/* Pick RX key according to TX key if any */
1833	key = rx->key;
1834	if (tx_key == key.active || tx_key == key.pending ||
1835	    tx_key == key.passive)
1836		goto decrypt;
1837
1838	/* Unknown key, let's try to align RX key(s) */
1839	if (tipc_crypto_key_try_align(rx, tx_key))
1840		goto decrypt;
1841
1842pick_tx:
1843	/* No key suitable? Try to pick one from TX... */
1844	aead = tipc_crypto_key_pick_tx(tx, rx, *skb, tx_key);
1845	if (aead)
1846		goto decrypt;
1847	goto exit;
1848
1849decrypt:
1850	rcu_read_lock();
1851	if (!aead)
1852		aead = tipc_aead_get(rx->aead[tx_key]);
1853	rc = tipc_aead_decrypt(net, aead, *skb, b);
1854	rcu_read_unlock();
1855
1856exit:
1857	stats = ((rx) ?: tx)->stats;
1858	switch (rc) {
1859	case 0:
1860		this_cpu_inc(stats->stat[STAT_OK]);
1861		break;
1862	case -EINPROGRESS:
1863	case -EBUSY:
1864		this_cpu_inc(stats->stat[STAT_ASYNC]);
1865		*skb = NULL;
1866		return rc;
1867	default:
1868		this_cpu_inc(stats->stat[STAT_NOK]);
1869		if (rc == -ENOKEY) {
1870			kfree_skb(*skb);
1871			*skb = NULL;
1872			if (rx) {
1873				/* Mark rx->nokey only if we dont have a
1874				 * pending received session key, nor a newer
1875				 * one i.e. in the next slot.
1876				 */
1877				n = key_next(tx_key);
1878				rx->nokey = !(rx->skey ||
1879					      rcu_access_pointer(rx->aead[n]));
1880				pr_debug_ratelimited("%s: nokey %d, key %d/%x\n",
1881						     rx->name, rx->nokey,
1882						     tx_key, rx->key.keys);
1883				tipc_node_put(rx->node);
1884			}
1885			this_cpu_inc(stats->stat[STAT_NOKEYS]);
1886			return rc;
1887		} else if (rc == -EBADMSG) {
1888			this_cpu_inc(stats->stat[STAT_BADMSGS]);
1889		}
1890		break;
1891	}
1892
1893	tipc_crypto_rcv_complete(net, aead, b, skb, rc);
1894	return rc;
1895}
1896
1897static void tipc_crypto_rcv_complete(struct net *net, struct tipc_aead *aead,
1898				     struct tipc_bearer *b,
1899				     struct sk_buff **skb, int err)
1900{
1901	struct tipc_skb_cb *skb_cb = TIPC_SKB_CB(*skb);
1902	struct tipc_crypto *rx = aead->crypto;
1903	struct tipc_aead *tmp = NULL;
1904	struct tipc_ehdr *ehdr;
1905	struct tipc_node *n;
1906
1907	/* Is this completed by TX? */
1908	if (unlikely(is_tx(aead->crypto))) {
1909		rx = skb_cb->tx_clone_ctx.rx;
1910		pr_debug("TX->RX(%s): err %d, aead %p, skb->next %p, flags %x\n",
1911			 (rx) ? tipc_node_get_id_str(rx->node) : "-", err, aead,
1912			 (*skb)->next, skb_cb->flags);
1913		pr_debug("skb_cb [recurs %d, last %p], tx->aead [%p %p %p]\n",
1914			 skb_cb->tx_clone_ctx.recurs, skb_cb->tx_clone_ctx.last,
1915			 aead->crypto->aead[1], aead->crypto->aead[2],
1916			 aead->crypto->aead[3]);
1917		if (unlikely(err)) {
1918			if (err == -EBADMSG && (*skb)->next)
1919				tipc_rcv(net, (*skb)->next, b);
1920			goto free_skb;
1921		}
1922
1923		if (likely((*skb)->next)) {
1924			kfree_skb((*skb)->next);
1925			(*skb)->next = NULL;
1926		}
1927		ehdr = (struct tipc_ehdr *)(*skb)->data;
1928		if (!rx) {
1929			WARN_ON(ehdr->user != LINK_CONFIG);
1930			n = tipc_node_create(net, 0, ehdr->id, 0xffffu, 0,
1931					     true);
1932			rx = tipc_node_crypto_rx(n);
1933			if (unlikely(!rx))
1934				goto free_skb;
1935		}
1936
1937		/* Ignore cloning if it was TX master key */
1938		if (ehdr->tx_key == KEY_MASTER)
1939			goto rcv;
1940		if (tipc_aead_clone(&tmp, aead) < 0)
1941			goto rcv;
1942		WARN_ON(!refcount_inc_not_zero(&tmp->refcnt));
1943		if (tipc_crypto_key_attach(rx, tmp, ehdr->tx_key, false) < 0) {
1944			tipc_aead_free(&tmp->rcu);
1945			goto rcv;
1946		}
1947		tipc_aead_put(aead);
1948		aead = tmp;
1949	}
1950
1951	if (unlikely(err)) {
1952		tipc_aead_users_dec((struct tipc_aead __force __rcu *)aead, INT_MIN);
1953		goto free_skb;
1954	}
1955
1956	/* Set the RX key's user */
1957	tipc_aead_users_set((struct tipc_aead __force __rcu *)aead, 1);
1958
1959	/* Mark this point, RX works */
1960	rx->timer1 = jiffies;
1961
1962rcv:
1963	/* Remove ehdr & auth. tag prior to tipc_rcv() */
1964	ehdr = (struct tipc_ehdr *)(*skb)->data;
1965
1966	/* Mark this point, RX passive still works */
1967	if (rx->key.passive && ehdr->tx_key == rx->key.passive)
1968		rx->timer2 = jiffies;
1969
1970	skb_reset_network_header(*skb);
1971	skb_pull(*skb, tipc_ehdr_size(ehdr));
1972	if (pskb_trim(*skb, (*skb)->len - aead->authsize))
1973		goto free_skb;
1974
1975	/* Validate TIPCv2 message */
1976	if (unlikely(!tipc_msg_validate(skb))) {
1977		pr_err_ratelimited("Packet dropped after decryption!\n");
1978		goto free_skb;
1979	}
1980
1981	/* Ok, everything's fine, try to synch own keys according to peers' */
1982	tipc_crypto_key_synch(rx, *skb);
1983
1984	/* Re-fetch skb cb as skb might be changed in tipc_msg_validate */
1985	skb_cb = TIPC_SKB_CB(*skb);
1986
1987	/* Mark skb decrypted */
1988	skb_cb->decrypted = 1;
1989
1990	/* Clear clone cxt if any */
1991	if (likely(!skb_cb->tx_clone_deferred))
1992		goto exit;
1993	skb_cb->tx_clone_deferred = 0;
1994	memset(&skb_cb->tx_clone_ctx, 0, sizeof(skb_cb->tx_clone_ctx));
1995	goto exit;
1996
1997free_skb:
1998	kfree_skb(*skb);
1999	*skb = NULL;
2000
2001exit:
2002	tipc_aead_put(aead);
2003	if (rx)
2004		tipc_node_put(rx->node);
2005}
2006
2007static void tipc_crypto_do_cmd(struct net *net, int cmd)
2008{
2009	struct tipc_net *tn = tipc_net(net);
2010	struct tipc_crypto *tx = tn->crypto_tx, *rx;
2011	struct list_head *p;
2012	unsigned int stat;
2013	int i, j, cpu;
2014	char buf[200];
2015
2016	/* Currently only one command is supported */
2017	switch (cmd) {
2018	case 0xfff1:
2019		goto print_stats;
2020	default:
2021		return;
2022	}
2023
2024print_stats:
2025	/* Print a header */
2026	pr_info("\n=============== TIPC Crypto Statistics ===============\n\n");
2027
2028	/* Print key status */
2029	pr_info("Key status:\n");
2030	pr_info("TX(%7.7s)\n%s", tipc_own_id_string(net),
2031		tipc_crypto_key_dump(tx, buf));
2032
2033	rcu_read_lock();
2034	for (p = tn->node_list.next; p != &tn->node_list; p = p->next) {
2035		rx = tipc_node_crypto_rx_by_list(p);
2036		pr_info("RX(%7.7s)\n%s", tipc_node_get_id_str(rx->node),
2037			tipc_crypto_key_dump(rx, buf));
2038	}
2039	rcu_read_unlock();
2040
2041	/* Print crypto statistics */
2042	for (i = 0, j = 0; i < MAX_STATS; i++)
2043		j += scnprintf(buf + j, 200 - j, "|%11s ", hstats[i]);
2044	pr_info("Counter     %s", buf);
2045
2046	memset(buf, '-', 115);
2047	buf[115] = '\0';
2048	pr_info("%s\n", buf);
2049
2050	j = scnprintf(buf, 200, "TX(%7.7s) ", tipc_own_id_string(net));
2051	for_each_possible_cpu(cpu) {
2052		for (i = 0; i < MAX_STATS; i++) {
2053			stat = per_cpu_ptr(tx->stats, cpu)->stat[i];
2054			j += scnprintf(buf + j, 200 - j, "|%11d ", stat);
2055		}
2056		pr_info("%s", buf);
2057		j = scnprintf(buf, 200, "%12s", " ");
2058	}
2059
2060	rcu_read_lock();
2061	for (p = tn->node_list.next; p != &tn->node_list; p = p->next) {
2062		rx = tipc_node_crypto_rx_by_list(p);
2063		j = scnprintf(buf, 200, "RX(%7.7s) ",
2064			      tipc_node_get_id_str(rx->node));
2065		for_each_possible_cpu(cpu) {
2066			for (i = 0; i < MAX_STATS; i++) {
2067				stat = per_cpu_ptr(rx->stats, cpu)->stat[i];
2068				j += scnprintf(buf + j, 200 - j, "|%11d ",
2069					       stat);
2070			}
2071			pr_info("%s", buf);
2072			j = scnprintf(buf, 200, "%12s", " ");
2073		}
2074	}
2075	rcu_read_unlock();
2076
2077	pr_info("\n======================== Done ========================\n");
2078}
2079
2080static char *tipc_crypto_key_dump(struct tipc_crypto *c, char *buf)
2081{
2082	struct tipc_key key = c->key;
2083	struct tipc_aead *aead;
2084	int k, i = 0;
2085	char *s;
2086
2087	for (k = KEY_MIN; k <= KEY_MAX; k++) {
2088		if (k == KEY_MASTER) {
2089			if (is_rx(c))
2090				continue;
2091			if (time_before(jiffies,
2092					c->timer2 + TIPC_TX_GRACE_PERIOD))
2093				s = "ACT";
2094			else
2095				s = "PAS";
2096		} else {
2097			if (k == key.passive)
2098				s = "PAS";
2099			else if (k == key.active)
2100				s = "ACT";
2101			else if (k == key.pending)
2102				s = "PEN";
2103			else
2104				s = "-";
2105		}
2106		i += scnprintf(buf + i, 200 - i, "\tKey%d: %s", k, s);
2107
2108		rcu_read_lock();
2109		aead = rcu_dereference(c->aead[k]);
2110		if (aead)
2111			i += scnprintf(buf + i, 200 - i,
2112				       "{\"0x...%s\", \"%s\"}/%d:%d",
2113				       aead->hint,
2114				       (aead->mode == CLUSTER_KEY) ? "c" : "p",
2115				       atomic_read(&aead->users),
2116				       refcount_read(&aead->refcnt));
2117		rcu_read_unlock();
2118		i += scnprintf(buf + i, 200 - i, "\n");
2119	}
2120
2121	if (is_rx(c))
2122		i += scnprintf(buf + i, 200 - i, "\tPeer RX active: %d\n",
2123			       atomic_read(&c->peer_rx_active));
2124
2125	return buf;
2126}
2127
2128static char *tipc_key_change_dump(struct tipc_key old, struct tipc_key new,
2129				  char *buf)
2130{
2131	struct tipc_key *key = &old;
2132	int k, i = 0;
2133	char *s;
2134
2135	/* Output format: "[%s %s %s] -> [%s %s %s]", max len = 32 */
2136again:
2137	i += scnprintf(buf + i, 32 - i, "[");
2138	for (k = KEY_1; k <= KEY_3; k++) {
2139		if (k == key->passive)
2140			s = "pas";
2141		else if (k == key->active)
2142			s = "act";
2143		else if (k == key->pending)
2144			s = "pen";
2145		else
2146			s = "-";
2147		i += scnprintf(buf + i, 32 - i,
2148			       (k != KEY_3) ? "%s " : "%s", s);
2149	}
2150	if (key != &new) {
2151		i += scnprintf(buf + i, 32 - i, "] -> ");
2152		key = &new;
2153		goto again;
2154	}
2155	i += scnprintf(buf + i, 32 - i, "]");
2156	return buf;
2157}
2158
2159/**
2160 * tipc_crypto_msg_rcv - Common 'MSG_CRYPTO' processing point
2161 * @net: the struct net
2162 * @skb: the receiving message buffer
2163 */
2164void tipc_crypto_msg_rcv(struct net *net, struct sk_buff *skb)
2165{
2166	struct tipc_crypto *rx;
2167	struct tipc_msg *hdr;
2168
2169	if (unlikely(skb_linearize(skb)))
2170		goto exit;
2171
2172	hdr = buf_msg(skb);
2173	rx = tipc_node_crypto_rx_by_addr(net, msg_prevnode(hdr));
2174	if (unlikely(!rx))
2175		goto exit;
2176
2177	switch (msg_type(hdr)) {
2178	case KEY_DISTR_MSG:
2179		if (tipc_crypto_key_rcv(rx, hdr))
2180			goto exit;
2181		break;
2182	default:
2183		break;
2184	}
2185
2186	tipc_node_put(rx->node);
2187
2188exit:
2189	kfree_skb(skb);
2190}
2191
2192/**
2193 * tipc_crypto_key_distr - Distribute a TX key
2194 * @tx: the TX crypto
2195 * @key: the key's index
2196 * @dest: the destination tipc node, = NULL if distributing to all nodes
2197 *
2198 * Return: 0 in case of success, otherwise < 0
2199 */
2200int tipc_crypto_key_distr(struct tipc_crypto *tx, u8 key,
2201			  struct tipc_node *dest)
2202{
2203	struct tipc_aead *aead;
2204	u32 dnode = tipc_node_get_addr(dest);
2205	int rc = -ENOKEY;
2206
2207	if (!sysctl_tipc_key_exchange_enabled)
2208		return 0;
2209
2210	if (key) {
2211		rcu_read_lock();
2212		aead = tipc_aead_get(tx->aead[key]);
2213		if (likely(aead)) {
2214			rc = tipc_crypto_key_xmit(tx->net, aead->key,
2215						  aead->gen, aead->mode,
2216						  dnode);
2217			tipc_aead_put(aead);
2218		}
2219		rcu_read_unlock();
2220	}
2221
2222	return rc;
2223}
2224
2225/**
2226 * tipc_crypto_key_xmit - Send a session key
2227 * @net: the struct net
2228 * @skey: the session key to be sent
2229 * @gen: the key's generation
2230 * @mode: the key's mode
2231 * @dnode: the destination node address, = 0 if broadcasting to all nodes
2232 *
2233 * The session key 'skey' is packed in a TIPC v2 'MSG_CRYPTO/KEY_DISTR_MSG'
2234 * as its data section, then xmit-ed through the uc/bc link.
2235 *
2236 * Return: 0 in case of success, otherwise < 0
2237 */
2238static int tipc_crypto_key_xmit(struct net *net, struct tipc_aead_key *skey,
2239				u16 gen, u8 mode, u32 dnode)
2240{
2241	struct sk_buff_head pkts;
2242	struct tipc_msg *hdr;
2243	struct sk_buff *skb;
2244	u16 size, cong_link_cnt;
2245	u8 *data;
2246	int rc;
2247
2248	size = tipc_aead_key_size(skey);
2249	skb = tipc_buf_acquire(INT_H_SIZE + size, GFP_ATOMIC);
2250	if (!skb)
2251		return -ENOMEM;
2252
2253	hdr = buf_msg(skb);
2254	tipc_msg_init(tipc_own_addr(net), hdr, MSG_CRYPTO, KEY_DISTR_MSG,
2255		      INT_H_SIZE, dnode);
2256	msg_set_size(hdr, INT_H_SIZE + size);
2257	msg_set_key_gen(hdr, gen);
2258	msg_set_key_mode(hdr, mode);
2259
2260	data = msg_data(hdr);
2261	*((__be32 *)(data + TIPC_AEAD_ALG_NAME)) = htonl(skey->keylen);
2262	memcpy(data, skey->alg_name, TIPC_AEAD_ALG_NAME);
2263	memcpy(data + TIPC_AEAD_ALG_NAME + sizeof(__be32), skey->key,
2264	       skey->keylen);
2265
2266	__skb_queue_head_init(&pkts);
2267	__skb_queue_tail(&pkts, skb);
2268	if (dnode)
2269		rc = tipc_node_xmit(net, &pkts, dnode, 0);
2270	else
2271		rc = tipc_bcast_xmit(net, &pkts, &cong_link_cnt);
2272
2273	return rc;
2274}
2275
2276/**
2277 * tipc_crypto_key_rcv - Receive a session key
2278 * @rx: the RX crypto
2279 * @hdr: the TIPC v2 message incl. the receiving session key in its data
2280 *
2281 * This function retrieves the session key in the message from peer, then
2282 * schedules a RX work to attach the key to the corresponding RX crypto.
2283 *
2284 * Return: "true" if the key has been scheduled for attaching, otherwise
2285 * "false".
2286 */
2287static bool tipc_crypto_key_rcv(struct tipc_crypto *rx, struct tipc_msg *hdr)
2288{
2289	struct tipc_crypto *tx = tipc_net(rx->net)->crypto_tx;
2290	struct tipc_aead_key *skey = NULL;
2291	u16 key_gen = msg_key_gen(hdr);
2292	u32 size = msg_data_sz(hdr);
2293	u8 *data = msg_data(hdr);
2294	unsigned int keylen;
2295
2296	/* Verify whether the size can exist in the packet */
2297	if (unlikely(size < sizeof(struct tipc_aead_key) + TIPC_AEAD_KEYLEN_MIN)) {
2298		pr_debug("%s: message data size is too small\n", rx->name);
2299		goto exit;
2300	}
2301
2302	keylen = ntohl(*((__be32 *)(data + TIPC_AEAD_ALG_NAME)));
2303
2304	/* Verify the supplied size values */
2305	if (unlikely(keylen > TIPC_AEAD_KEY_SIZE_MAX ||
2306		     size != keylen + sizeof(struct tipc_aead_key))) {
2307		pr_debug("%s: invalid MSG_CRYPTO key size\n", rx->name);
2308		goto exit;
2309	}
2310
2311	spin_lock(&rx->lock);
2312	if (unlikely(rx->skey || (key_gen == rx->key_gen && rx->key.keys))) {
2313		pr_err("%s: key existed <%p>, gen %d vs %d\n", rx->name,
2314		       rx->skey, key_gen, rx->key_gen);
2315		goto exit_unlock;
2316	}
2317
2318	/* Allocate memory for the key */
2319	skey = kmalloc(size, GFP_ATOMIC);
2320	if (unlikely(!skey)) {
2321		pr_err("%s: unable to allocate memory for skey\n", rx->name);
2322		goto exit_unlock;
2323	}
2324
2325	/* Copy key from msg data */
2326	skey->keylen = keylen;
2327	memcpy(skey->alg_name, data, TIPC_AEAD_ALG_NAME);
2328	memcpy(skey->key, data + TIPC_AEAD_ALG_NAME + sizeof(__be32),
2329	       skey->keylen);
2330
2331	rx->key_gen = key_gen;
2332	rx->skey_mode = msg_key_mode(hdr);
2333	rx->skey = skey;
2334	rx->nokey = 0;
2335	mb(); /* for nokey flag */
2336
2337exit_unlock:
2338	spin_unlock(&rx->lock);
2339
2340exit:
2341	/* Schedule the key attaching on this crypto */
2342	if (likely(skey && queue_delayed_work(tx->wq, &rx->work, 0)))
2343		return true;
2344
2345	return false;
2346}
2347
2348/**
2349 * tipc_crypto_work_rx - Scheduled RX works handler
2350 * @work: the struct RX work
2351 *
2352 * The function processes the previous scheduled works i.e. distributing TX key
2353 * or attaching a received session key on RX crypto.
2354 */
2355static void tipc_crypto_work_rx(struct work_struct *work)
2356{
2357	struct delayed_work *dwork = to_delayed_work(work);
2358	struct tipc_crypto *rx = container_of(dwork, struct tipc_crypto, work);
2359	struct tipc_crypto *tx = tipc_net(rx->net)->crypto_tx;
2360	unsigned long delay = msecs_to_jiffies(5000);
2361	bool resched = false;
2362	u8 key;
2363	int rc;
2364
2365	/* Case 1: Distribute TX key to peer if scheduled */
2366	if (atomic_cmpxchg(&rx->key_distr,
2367			   KEY_DISTR_SCHED,
2368			   KEY_DISTR_COMPL) == KEY_DISTR_SCHED) {
2369		/* Always pick the newest one for distributing */
2370		key = tx->key.pending ?: tx->key.active;
2371		rc = tipc_crypto_key_distr(tx, key, rx->node);
2372		if (unlikely(rc))
2373			pr_warn("%s: unable to distr key[%d] to %s, err %d\n",
2374				tx->name, key, tipc_node_get_id_str(rx->node),
2375				rc);
2376
2377		/* Sched for key_distr releasing */
2378		resched = true;
2379	} else {
2380		atomic_cmpxchg(&rx->key_distr, KEY_DISTR_COMPL, 0);
2381	}
2382
2383	/* Case 2: Attach a pending received session key from peer if any */
2384	if (rx->skey) {
2385		rc = tipc_crypto_key_init(rx, rx->skey, rx->skey_mode, false);
2386		if (unlikely(rc < 0))
2387			pr_warn("%s: unable to attach received skey, err %d\n",
2388				rx->name, rc);
2389		switch (rc) {
2390		case -EBUSY:
2391		case -ENOMEM:
2392			/* Resched the key attaching */
2393			resched = true;
2394			break;
2395		default:
2396			synchronize_rcu();
2397			kfree_sensitive(rx->skey);
2398			rx->skey = NULL;
2399			break;
2400		}
2401	}
2402
2403	if (resched && queue_delayed_work(tx->wq, &rx->work, delay))
2404		return;
2405
2406	tipc_node_put(rx->node);
2407}
2408
2409/**
2410 * tipc_crypto_rekeying_sched - (Re)schedule rekeying w/o new interval
2411 * @tx: TX crypto
2412 * @changed: if the rekeying needs to be rescheduled with new interval
2413 * @new_intv: new rekeying interval (when "changed" = true)
2414 */
2415void tipc_crypto_rekeying_sched(struct tipc_crypto *tx, bool changed,
2416				u32 new_intv)
2417{
2418	unsigned long delay;
2419	bool now = false;
2420
2421	if (changed) {
2422		if (new_intv == TIPC_REKEYING_NOW)
2423			now = true;
2424		else
2425			tx->rekeying_intv = new_intv;
2426		cancel_delayed_work_sync(&tx->work);
2427	}
2428
2429	if (tx->rekeying_intv || now) {
2430		delay = (now) ? 0 : tx->rekeying_intv * 60 * 1000;
2431		queue_delayed_work(tx->wq, &tx->work, msecs_to_jiffies(delay));
2432	}
2433}
2434
2435/**
2436 * tipc_crypto_work_tx - Scheduled TX works handler
2437 * @work: the struct TX work
2438 *
2439 * The function processes the previous scheduled work, i.e. key rekeying, by
2440 * generating a new session key based on current one, then attaching it to the
2441 * TX crypto and finally distributing it to peers. It also re-schedules the
2442 * rekeying if needed.
2443 */
2444static void tipc_crypto_work_tx(struct work_struct *work)
2445{
2446	struct delayed_work *dwork = to_delayed_work(work);
2447	struct tipc_crypto *tx = container_of(dwork, struct tipc_crypto, work);
2448	struct tipc_aead_key *skey = NULL;
2449	struct tipc_key key = tx->key;
2450	struct tipc_aead *aead;
2451	int rc = -ENOMEM;
2452
2453	if (unlikely(key.pending))
2454		goto resched;
2455
2456	/* Take current key as a template */
2457	rcu_read_lock();
2458	aead = rcu_dereference(tx->aead[key.active ?: KEY_MASTER]);
2459	if (unlikely(!aead)) {
2460		rcu_read_unlock();
2461		/* At least one key should exist for securing */
2462		return;
2463	}
2464
2465	/* Lets duplicate it first */
2466	skey = kmemdup(aead->key, tipc_aead_key_size(aead->key), GFP_ATOMIC);
2467	rcu_read_unlock();
2468
2469	/* Now, generate new key, initiate & distribute it */
2470	if (likely(skey)) {
2471		rc = tipc_aead_key_generate(skey) ?:
2472		     tipc_crypto_key_init(tx, skey, PER_NODE_KEY, false);
2473		if (likely(rc > 0))
2474			rc = tipc_crypto_key_distr(tx, rc, NULL);
2475		kfree_sensitive(skey);
2476	}
2477
2478	if (unlikely(rc))
2479		pr_warn_ratelimited("%s: rekeying returns %d\n", tx->name, rc);
2480
2481resched:
2482	/* Re-schedule rekeying if any */
2483	tipc_crypto_rekeying_sched(tx, false, 0);
2484}