chacha20poly1305.c

  1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2/*
  3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  4 *
  5 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
  6 *
  7 * Information: https://tools.ietf.org/html/rfc8439
  8 */
  9
 10#include <crypto/chacha.h>
 11#include <crypto/chacha20poly1305.h>
 12#include <crypto/poly1305.h>
 13#include <crypto/utils.h>
 14#include <linux/export.h>
 15#include <linux/init.h>
 16#include <linux/kernel.h>
 17#include <linux/mm.h>
 18#include <linux/module.h>
 19#include <linux/unaligned.h>
 20
 21static void chacha_load_key(u32 *k, const u8 *in)
 22{
 23	k[0] = get_unaligned_le32(in);
 24	k[1] = get_unaligned_le32(in + 4);
 25	k[2] = get_unaligned_le32(in + 8);
 26	k[3] = get_unaligned_le32(in + 12);
 27	k[4] = get_unaligned_le32(in + 16);
 28	k[5] = get_unaligned_le32(in + 20);
 29	k[6] = get_unaligned_le32(in + 24);
 30	k[7] = get_unaligned_le32(in + 28);
 31}
 32
 33static void xchacha_init(struct chacha_state *chacha_state,
 34			 const u8 *key, const u8 *nonce)
 35{
 36	u32 k[CHACHA_KEY_WORDS];
 37	u8 iv[CHACHA_IV_SIZE];
 38
 39	memset(iv, 0, 8);
 40	memcpy(iv + 8, nonce + 16, 8);
 41
 42	chacha_load_key(k, key);
 43
 44	/* Compute the subkey given the original key and first 128 nonce bits */
 45	chacha_init(chacha_state, k, nonce);
 46	hchacha_block(chacha_state, k, 20);
 47
 48	chacha_init(chacha_state, k, iv);
 49
 50	memzero_explicit(k, sizeof(k));
 51	memzero_explicit(iv, sizeof(iv));
 52}
 53
 54static void
 55__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 56			   const u8 *ad, const size_t ad_len,
 57			   struct chacha_state *chacha_state)
 58{
 59	const u8 *pad0 = page_address(ZERO_PAGE(0));
 60	struct poly1305_desc_ctx poly1305_state;
 61	union {
 62		u8 block0[POLY1305_KEY_SIZE];
 63		__le64 lens[2];
 64	} b;
 65
 66	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
 67	poly1305_init(&poly1305_state, b.block0);
 68
 69	poly1305_update(&poly1305_state, ad, ad_len);
 70	if (ad_len & 0xf)
 71		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
 72
 73	chacha20_crypt(chacha_state, dst, src, src_len);
 74
 75	poly1305_update(&poly1305_state, dst, src_len);
 76	if (src_len & 0xf)
 77		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
 78
 79	b.lens[0] = cpu_to_le64(ad_len);
 80	b.lens[1] = cpu_to_le64(src_len);
 81	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
 82
 83	poly1305_final(&poly1305_state, dst + src_len);
 84
 85	chacha_zeroize_state(chacha_state);
 86	memzero_explicit(&b, sizeof(b));
 87}
 88
 89void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
 90			      const u8 *ad, const size_t ad_len,
 91			      const u64 nonce,
 92			      const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
 93{
 94	struct chacha_state chacha_state;
 95	u32 k[CHACHA_KEY_WORDS];
 96	__le64 iv[2];
 97
 98	chacha_load_key(k, key);
 99
100	iv[0] = 0;
101	iv[1] = cpu_to_le64(nonce);
102
103	chacha_init(&chacha_state, k, (u8 *)iv);
104	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
105				   &chacha_state);
106
107	memzero_explicit(iv, sizeof(iv));
108	memzero_explicit(k, sizeof(k));
109}
110EXPORT_SYMBOL(chacha20poly1305_encrypt);
111
112void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113			       const u8 *ad, const size_t ad_len,
114			       const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE],
115			       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
116{
117	struct chacha_state chacha_state;
118
119	xchacha_init(&chacha_state, key, nonce);
120	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len,
121				   &chacha_state);
122}
123EXPORT_SYMBOL(xchacha20poly1305_encrypt);
124
125static bool
126__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
127			   const u8 *ad, const size_t ad_len,
128			   struct chacha_state *chacha_state)
129{
130	const u8 *pad0 = page_address(ZERO_PAGE(0));
131	struct poly1305_desc_ctx poly1305_state;
132	size_t dst_len;
133	int ret;
134	union {
135		u8 block0[POLY1305_KEY_SIZE];
136		u8 mac[POLY1305_DIGEST_SIZE];
137		__le64 lens[2];
138	} b;
139
140	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
141		return false;
142
143	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
144	poly1305_init(&poly1305_state, b.block0);
145
146	poly1305_update(&poly1305_state, ad, ad_len);
147	if (ad_len & 0xf)
148		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
149
150	dst_len = src_len - POLY1305_DIGEST_SIZE;
151	poly1305_update(&poly1305_state, src, dst_len);
152	if (dst_len & 0xf)
153		poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
154
155	b.lens[0] = cpu_to_le64(ad_len);
156	b.lens[1] = cpu_to_le64(dst_len);
157	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
158
159	poly1305_final(&poly1305_state, b.mac);
160
161	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
162	if (likely(!ret))
163		chacha20_crypt(chacha_state, dst, src, dst_len);
164
165	memzero_explicit(&b, sizeof(b));
166
167	return !ret;
168}
169
170bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
171			      const u8 *ad, const size_t ad_len,
172			      const u64 nonce,
173			      const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
174{
175	struct chacha_state chacha_state;
176	u32 k[CHACHA_KEY_WORDS];
177	__le64 iv[2];
178	bool ret;
179
180	chacha_load_key(k, key);
181
182	iv[0] = 0;
183	iv[1] = cpu_to_le64(nonce);
184
185	chacha_init(&chacha_state, k, (u8 *)iv);
186	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
187					 &chacha_state);
188
189	chacha_zeroize_state(&chacha_state);
190	memzero_explicit(iv, sizeof(iv));
191	memzero_explicit(k, sizeof(k));
192	return ret;
193}
194EXPORT_SYMBOL(chacha20poly1305_decrypt);
195
196bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
197			       const u8 *ad, const size_t ad_len,
198			       const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE],
199			       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
200{
201	struct chacha_state chacha_state;
202
203	xchacha_init(&chacha_state, key, nonce);
204	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
205					  &chacha_state);
206}
207EXPORT_SYMBOL(xchacha20poly1305_decrypt);
208
209static
210bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
211				       const size_t src_len,
212				       const u8 *ad, const size_t ad_len,
213				       const u64 nonce,
214				       const u8 key[at_least CHACHA20POLY1305_KEY_SIZE],
215				       int encrypt)
216{
217	const u8 *pad0 = page_address(ZERO_PAGE(0));
218	struct poly1305_desc_ctx poly1305_state;
219	struct chacha_state chacha_state;
220	struct sg_mapping_iter miter;
221	size_t partial = 0;
222	unsigned int flags;
223	bool ret = true;
224	int sl;
225	union {
226		struct {
227			u32 k[CHACHA_KEY_WORDS];
228			__le64 iv[2];
229		};
230		u8 block0[POLY1305_KEY_SIZE];
231		u8 chacha_stream[CHACHA_BLOCK_SIZE];
232		struct {
233			u8 mac[2][POLY1305_DIGEST_SIZE];
234		};
235		__le64 lens[2];
236	} b __aligned(16);
237
238	if (WARN_ON(src_len > INT_MAX))
239		return false;
240
241	chacha_load_key(b.k, key);
242
243	b.iv[0] = 0;
244	b.iv[1] = cpu_to_le64(nonce);
245
246	chacha_init(&chacha_state, b.k, (u8 *)b.iv);
247	chacha20_crypt(&chacha_state, b.block0, pad0, sizeof(b.block0));
248	poly1305_init(&poly1305_state, b.block0);
249
250	if (unlikely(ad_len)) {
251		poly1305_update(&poly1305_state, ad, ad_len);
252		if (ad_len & 0xf)
253			poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
254	}
255
256	flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
257
258	sg_miter_start(&miter, src, sg_nents(src), flags);
259
260	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
261		u8 *addr = miter.addr;
262		size_t length = min_t(size_t, sl, miter.length);
263
264		if (!encrypt)
265			poly1305_update(&poly1305_state, addr, length);
266
267		if (unlikely(partial)) {
268			size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
269
270			crypto_xor(addr, b.chacha_stream + partial, l);
271			partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
272
273			addr += l;
274			length -= l;
275		}
276
277		if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
278			size_t l = length;
279
280			if (unlikely(length < sl))
281				l &= ~(CHACHA_BLOCK_SIZE - 1);
282			chacha20_crypt(&chacha_state, addr, addr, l);
283			addr += l;
284			length -= l;
285		}
286
287		if (unlikely(length > 0)) {
288			chacha20_crypt(&chacha_state, b.chacha_stream, pad0,
289				       CHACHA_BLOCK_SIZE);
290			crypto_xor(addr, b.chacha_stream, length);
291			partial = length;
292		}
293
294		if (encrypt)
295			poly1305_update(&poly1305_state, miter.addr,
296					min_t(size_t, sl, miter.length));
297	}
298
299	if (src_len & 0xf)
300		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
301
302	b.lens[0] = cpu_to_le64(ad_len);
303	b.lens[1] = cpu_to_le64(src_len);
304	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
305
306	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
307		if (encrypt) {
308			poly1305_final(&poly1305_state,
309				       miter.addr + miter.length + sl);
310			ret = true;
311		} else {
312			poly1305_final(&poly1305_state, b.mac[0]);
313			ret = !crypto_memneq(b.mac[0],
314					     miter.addr + miter.length + sl,
315					     POLY1305_DIGEST_SIZE);
316		}
317	}
318
319	sg_miter_stop(&miter);
320
321	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
322		poly1305_final(&poly1305_state, b.mac[1]);
323		sg_copy_buffer(src, sg_nents(src), b.mac[encrypt],
324			       sizeof(b.mac[1]), src_len, !encrypt);
325		ret = encrypt ||
326		      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
327	}
328
329	chacha_zeroize_state(&chacha_state);
330	memzero_explicit(&b, sizeof(b));
331
332	return ret;
333}
334
335bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
336					 const u8 *ad, const size_t ad_len,
337					 const u64 nonce,
338					 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
339{
340	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
341						 nonce, key, 1);
342}
343EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
344
345bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
346					 const u8 *ad, const size_t ad_len,
347					 const u64 nonce,
348					 const u8 key[at_least CHACHA20POLY1305_KEY_SIZE])
349{
350	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
351		return false;
352
353	return chacha20poly1305_crypt_sg_inplace(src,
354						 src_len - POLY1305_DIGEST_SIZE,
355						 ad, ad_len, nonce, key, 0);
356}
357EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
358
359static int __init chacha20poly1305_init(void)
360{
361	if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) &&
362	    WARN_ON(!chacha20poly1305_selftest()))
363		return -ENODEV;
364	return 0;
365}
366
367static void __exit chacha20poly1305_exit(void)
368{
369}
370
371module_init(chacha20poly1305_init);
372module_exit(chacha20poly1305_exit);
373MODULE_LICENSE("GPL v2");
374MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
375MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");