scompress.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Synchronous Compression operations
  4 *
  5 * Copyright 2015 LG Electronics Inc.
  6 * Copyright (c) 2016, Intel Corporation
  7 * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
  8 */
  9
 10#include <crypto/internal/scompress.h>
 11#include <crypto/scatterwalk.h>
 12#include <linux/cpumask.h>
 13#include <linux/cryptouser.h>
 14#include <linux/err.h>
 15#include <linux/highmem.h>
 16#include <linux/kernel.h>
 17#include <linux/module.h>
 18#include <linux/overflow.h>
 19#include <linux/scatterlist.h>
 20#include <linux/seq_file.h>
 21#include <linux/slab.h>
 22#include <linux/string.h>
 23#include <linux/workqueue.h>
 24#include <net/netlink.h>
 25
 26#include "compress.h"
 27
 28struct scomp_scratch {
 29	spinlock_t	lock;
 30	union {
 31		void	*src;
 32		unsigned long saddr;
 33	};
 34};
 35
 36static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = {
 37	.lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock),
 38};
 39
 40static const struct crypto_type crypto_scomp_type;
 41static int scomp_scratch_users;
 42static DEFINE_MUTEX(scomp_lock);
 43
 44static cpumask_t scomp_scratch_want;
 45static void scomp_scratch_workfn(struct work_struct *work);
 46static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn);
 47
 48static int __maybe_unused crypto_scomp_report(
 49	struct sk_buff *skb, struct crypto_alg *alg)
 50{
 51	struct crypto_report_comp rscomp;
 52
 53	memset(&rscomp, 0, sizeof(rscomp));
 54
 55	strscpy(rscomp.type, "scomp", sizeof(rscomp.type));
 56
 57	return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
 58		       sizeof(rscomp), &rscomp);
 59}
 60
 61static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
 62	__maybe_unused;
 63
 64static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
 65{
 66	seq_puts(m, "type         : scomp\n");
 67}
 68
 69static void crypto_scomp_free_scratches(void)
 70{
 71	struct scomp_scratch *scratch;
 72	int i;
 73
 74	for_each_possible_cpu(i) {
 75		scratch = per_cpu_ptr(&scomp_scratch, i);
 76
 77		free_page(scratch->saddr);
 78		scratch->src = NULL;
 79	}
 80}
 81
 82static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu)
 83{
 84	int node = cpu_to_node(cpu);
 85	struct page *page;
 86
 87	page = alloc_pages_node(node, GFP_KERNEL, 0);
 88	if (!page)
 89		return -ENOMEM;
 90	spin_lock_bh(&scratch->lock);
 91	scratch->src = page_address(page);
 92	spin_unlock_bh(&scratch->lock);
 93	return 0;
 94}
 95
 96static void scomp_scratch_workfn(struct work_struct *work)
 97{
 98	int cpu;
 99
100	for_each_cpu(cpu, &scomp_scratch_want) {
101		struct scomp_scratch *scratch;
102
103		scratch = per_cpu_ptr(&scomp_scratch, cpu);
104		if (scratch->src)
105			continue;
106		if (scomp_alloc_scratch(scratch, cpu))
107			break;
108
109		cpumask_clear_cpu(cpu, &scomp_scratch_want);
110	}
111}
112
113static int crypto_scomp_alloc_scratches(void)
114{
115	unsigned int i = cpumask_first(cpu_possible_mask);
116	struct scomp_scratch *scratch;
117
118	scratch = per_cpu_ptr(&scomp_scratch, i);
119	return scomp_alloc_scratch(scratch, i);
120}
121
122static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
123{
124	struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm));
125	int ret = 0;
126
127	mutex_lock(&scomp_lock);
128	ret = crypto_acomp_alloc_streams(&alg->streams);
129	if (ret)
130		goto unlock;
131	if (!scomp_scratch_users++) {
132		ret = crypto_scomp_alloc_scratches();
133		if (ret)
134			scomp_scratch_users--;
135	}
136unlock:
137	mutex_unlock(&scomp_lock);
138
139	return ret;
140}
141
142static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch)
143{
144	int cpu = raw_smp_processor_id();
145	struct scomp_scratch *scratch;
146
147	scratch = per_cpu_ptr(&scomp_scratch, cpu);
148	spin_lock(&scratch->lock);
149	if (likely(scratch->src))
150		return scratch;
151	spin_unlock(&scratch->lock);
152
153	cpumask_set_cpu(cpu, &scomp_scratch_want);
154	schedule_work(&scomp_scratch_work);
155
156	scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask));
157	spin_lock(&scratch->lock);
158	return scratch;
159}
160
161static inline void scomp_unlock_scratch(struct scomp_scratch *scratch)
162	__releases(scratch)
163{
164	spin_unlock(&scratch->lock);
165}
166
167static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
168{
169	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
170	struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm);
171	bool src_isvirt = acomp_request_src_isvirt(req);
172	bool dst_isvirt = acomp_request_dst_isvirt(req);
173	struct crypto_scomp *scomp = *tfm_ctx;
174	struct crypto_acomp_stream *stream;
175	struct scomp_scratch *scratch;
176	unsigned int slen = req->slen;
177	unsigned int dlen = req->dlen;
178	struct page *spage, *dpage;
179	unsigned int n;
180	const u8 *src;
181	size_t soff;
182	size_t doff;
183	u8 *dst;
184	int ret;
185
186	if (!req->src || !slen)
187		return -EINVAL;
188
189	if (!req->dst || !dlen)
190		return -EINVAL;
191
192	if (dst_isvirt)
193		dst = req->dvirt;
194	else {
195		if (dlen <= req->dst->length) {
196			dpage = sg_page(req->dst);
197			doff = req->dst->offset;
198		} else
199			return -ENOSYS;
200
201		dpage += doff / PAGE_SIZE;
202		doff = offset_in_page(doff);
203
204		n = (dlen - 1) / PAGE_SIZE;
205		n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE;
206		if (PageHighMem(dpage + n) &&
207		    size_add(doff, dlen) > PAGE_SIZE)
208			return -ENOSYS;
209		dst = kmap_local_page(dpage) + doff;
210	}
211
212	if (src_isvirt)
213		src = req->svirt;
214	else {
215		src = NULL;
216		do {
217			if (slen <= req->src->length) {
218				spage = sg_page(req->src);
219				soff = req->src->offset;
220			} else
221				break;
222
223			spage = spage + soff / PAGE_SIZE;
224			soff = offset_in_page(soff);
225
226			n = (slen - 1) / PAGE_SIZE;
227			n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE;
228			if (PageHighMem(spage + n) &&
229			    size_add(soff, slen) > PAGE_SIZE)
230				break;
231			src = kmap_local_page(spage) + soff;
232		} while (0);
233	}
234
235	stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams);
236
237	if (!src_isvirt && !src) {
238		const u8 *src;
239
240		scratch = scomp_lock_scratch();
241		src = scratch->src;
242		memcpy_from_sglist(scratch->src, req->src, 0, slen);
243
244		if (dir)
245			ret = crypto_scomp_compress(scomp, src, slen,
246						    dst, &dlen, stream->ctx);
247		else
248			ret = crypto_scomp_decompress(scomp, src, slen,
249						      dst, &dlen, stream->ctx);
250
251		scomp_unlock_scratch(scratch);
252	} else if (dir)
253		ret = crypto_scomp_compress(scomp, src, slen,
254					    dst, &dlen, stream->ctx);
255	else
256		ret = crypto_scomp_decompress(scomp, src, slen,
257					      dst, &dlen, stream->ctx);
258
259	crypto_acomp_unlock_stream_bh(stream);
260
261	req->dlen = dlen;
262
263	if (!src_isvirt && src)
264		kunmap_local(src);
265	if (!dst_isvirt) {
266		kunmap_local(dst);
267		dlen += doff;
268		for (;;) {
269			flush_dcache_page(dpage);
270			if (dlen <= PAGE_SIZE)
271				break;
272			dlen -= PAGE_SIZE;
273			dpage++;
274		}
275	}
276
277	return ret;
278}
279
280static int scomp_acomp_compress(struct acomp_req *req)
281{
282	return scomp_acomp_comp_decomp(req, 1);
283}
284
285static int scomp_acomp_decompress(struct acomp_req *req)
286{
287	return scomp_acomp_comp_decomp(req, 0);
288}
289
290static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
291{
292	struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
293
294	crypto_free_scomp(*ctx);
295
296	flush_work(&scomp_scratch_work);
297	mutex_lock(&scomp_lock);
298	if (!--scomp_scratch_users)
299		crypto_scomp_free_scratches();
300	mutex_unlock(&scomp_lock);
301}
302
303int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
304{
305	struct crypto_alg *calg = tfm->__crt_alg;
306	struct crypto_acomp *crt = __crypto_acomp_tfm(tfm);
307	struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
308	struct crypto_scomp *scomp;
309
310	if (!crypto_mod_get(calg))
311		return -EAGAIN;
312
313	scomp = crypto_create_tfm(calg, &crypto_scomp_type);
314	if (IS_ERR(scomp)) {
315		crypto_mod_put(calg);
316		return PTR_ERR(scomp);
317	}
318
319	*ctx = scomp;
320	tfm->exit = crypto_exit_scomp_ops_async;
321
322	crt->compress = scomp_acomp_compress;
323	crt->decompress = scomp_acomp_decompress;
324
325	return 0;
326}
327
328static void crypto_scomp_destroy(struct crypto_alg *alg)
329{
330	struct scomp_alg *scomp = __crypto_scomp_alg(alg);
331
332	crypto_acomp_free_streams(&scomp->streams);
333}
334
335static const struct crypto_type crypto_scomp_type = {
336	.extsize = crypto_alg_extsize,
337	.init_tfm = crypto_scomp_init_tfm,
338	.destroy = crypto_scomp_destroy,
339#ifdef CONFIG_PROC_FS
340	.show = crypto_scomp_show,
341#endif
342#if IS_ENABLED(CONFIG_CRYPTO_USER)
343	.report = crypto_scomp_report,
344#endif
345	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
346	.maskset = CRYPTO_ALG_TYPE_MASK,
347	.type = CRYPTO_ALG_TYPE_SCOMPRESS,
348	.tfmsize = offsetof(struct crypto_scomp, base),
349	.algsize = offsetof(struct scomp_alg, base),
350};
351
352static void scomp_prepare_alg(struct scomp_alg *alg)
353{
354	struct crypto_alg *base = &alg->calg.base;
355
356	comp_prepare_alg(&alg->calg);
357
358	base->cra_flags |= CRYPTO_ALG_REQ_VIRT;
359}
360
361int crypto_register_scomp(struct scomp_alg *alg)
362{
363	struct crypto_alg *base = &alg->calg.base;
364
365	scomp_prepare_alg(alg);
366
367	base->cra_type = &crypto_scomp_type;
368	base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
369
370	return crypto_register_alg(base);
371}
372EXPORT_SYMBOL_GPL(crypto_register_scomp);
373
374void crypto_unregister_scomp(struct scomp_alg *alg)
375{
376	crypto_unregister_alg(&alg->base);
377}
378EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
379
380int crypto_register_scomps(struct scomp_alg *algs, int count)
381{
382	int i, ret;
383
384	for (i = 0; i < count; i++) {
385		ret = crypto_register_scomp(&algs[i]);
386		if (ret)
387			goto err;
388	}
389
390	return 0;
391
392err:
393	for (--i; i >= 0; --i)
394		crypto_unregister_scomp(&algs[i]);
395
396	return ret;
397}
398EXPORT_SYMBOL_GPL(crypto_register_scomps);
399
400void crypto_unregister_scomps(struct scomp_alg *algs, int count)
401{
402	int i;
403
404	for (i = count - 1; i >= 0; --i)
405		crypto_unregister_scomp(&algs[i]);
406}
407EXPORT_SYMBOL_GPL(crypto_unregister_scomps);
408
409MODULE_LICENSE("GPL");
410MODULE_DESCRIPTION("Synchronous compression type");