acompress.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * Asynchronous Compression operations
  4 *
  5 * Copyright (c) 2016, Intel Corporation
  6 * Authors: Weigang Li <weigang.li@intel.com>
  7 *          Giovanni Cabiddu <giovanni.cabiddu@intel.com>
  8 */
  9
 10#include <crypto/internal/acompress.h>
 11#include <crypto/scatterwalk.h>
 12#include <linux/cryptouser.h>
 13#include <linux/cpumask.h>
 14#include <linux/err.h>
 15#include <linux/kernel.h>
 16#include <linux/module.h>
 17#include <linux/percpu.h>
 18#include <linux/scatterlist.h>
 19#include <linux/sched.h>
 20#include <linux/seq_file.h>
 21#include <linux/smp.h>
 22#include <linux/spinlock.h>
 23#include <linux/string.h>
 24#include <linux/workqueue.h>
 25#include <net/netlink.h>
 26
 27#include "compress.h"
 28
 29struct crypto_scomp;
 30
 31enum {
 32	ACOMP_WALK_SLEEP = 1 << 0,
 33	ACOMP_WALK_SRC_LINEAR = 1 << 1,
 34	ACOMP_WALK_DST_LINEAR = 1 << 2,
 35};
 36
 37static const struct crypto_type crypto_acomp_type;
 38
 39static void acomp_reqchain_done(void *data, int err);
 40
 41static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
 42{
 43	return container_of(alg, struct acomp_alg, calg.base);
 44}
 45
 46static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
 47{
 48	return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
 49}
 50
 51static int __maybe_unused crypto_acomp_report(
 52	struct sk_buff *skb, struct crypto_alg *alg)
 53{
 54	struct crypto_report_acomp racomp;
 55
 56	memset(&racomp, 0, sizeof(racomp));
 57
 58	strscpy(racomp.type, "acomp", sizeof(racomp.type));
 59
 60	return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
 61}
 62
 63static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
 64	__maybe_unused;
 65
 66static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
 67{
 68	seq_puts(m, "type         : acomp\n");
 69}
 70
 71static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
 72{
 73	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
 74	struct acomp_alg *alg = crypto_acomp_alg(acomp);
 75
 76	if (alg->exit)
 77		alg->exit(acomp);
 78
 79	if (acomp_is_async(acomp))
 80		crypto_free_acomp(crypto_acomp_fb(acomp));
 81}
 82
 83static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
 84{
 85	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
 86	struct acomp_alg *alg = crypto_acomp_alg(acomp);
 87	struct crypto_acomp *fb = NULL;
 88	int err;
 89
 90	if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
 91		return crypto_init_scomp_ops_async(tfm);
 92
 93	if (acomp_is_async(acomp)) {
 94		fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0,
 95					CRYPTO_ALG_ASYNC);
 96		if (IS_ERR(fb))
 97			return PTR_ERR(fb);
 98
 99		err = -EINVAL;
100		if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE)
101			goto out_free_fb;
102
103		tfm->fb = crypto_acomp_tfm(fb);
104	}
105
106	acomp->compress = alg->compress;
107	acomp->decompress = alg->decompress;
108	acomp->reqsize = alg->base.cra_reqsize;
109
110	acomp->base.exit = crypto_acomp_exit_tfm;
111
112	if (!alg->init)
113		return 0;
114
115	err = alg->init(acomp);
116	if (err)
117		goto out_free_fb;
118
119	return 0;
120
121out_free_fb:
122	crypto_free_acomp(fb);
123	return err;
124}
125
126static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
127{
128	int extsize = crypto_alg_extsize(alg);
129
130	if (alg->cra_type != &crypto_acomp_type)
131		extsize += sizeof(struct crypto_scomp *);
132
133	return extsize;
134}
135
136static const struct crypto_type crypto_acomp_type = {
137	.extsize = crypto_acomp_extsize,
138	.init_tfm = crypto_acomp_init_tfm,
139#ifdef CONFIG_PROC_FS
140	.show = crypto_acomp_show,
141#endif
142#if IS_ENABLED(CONFIG_CRYPTO_USER)
143	.report = crypto_acomp_report,
144#endif
145	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
146	.maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
147	.type = CRYPTO_ALG_TYPE_ACOMPRESS,
148	.tfmsize = offsetof(struct crypto_acomp, base),
149	.algsize = offsetof(struct acomp_alg, base),
150};
151
152struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
153					u32 mask)
154{
155	return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
156}
157EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
158
159struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
160					u32 mask, int node)
161{
162	return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask,
163				node);
164}
165EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
166
167static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
168{
169	struct acomp_req_chain *state = &req->chain;
170
171	state->compl = req->base.complete;
172	state->data = req->base.data;
173	req->base.complete = cplt;
174	req->base.data = state;
175}
176
177static void acomp_restore_req(struct acomp_req *req)
178{
179	struct acomp_req_chain *state = req->base.data;
180
181	req->base.complete = state->compl;
182	req->base.data = state->data;
183}
184
185static void acomp_reqchain_virt(struct acomp_req *req)
186{
187	struct acomp_req_chain *state = &req->chain;
188	unsigned int slen = req->slen;
189	unsigned int dlen = req->dlen;
190
191	if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
192		acomp_request_set_src_dma(req, state->src, slen);
193	if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
194		acomp_request_set_dst_dma(req, state->dst, dlen);
195}
196
197static void acomp_virt_to_sg(struct acomp_req *req)
198{
199	struct acomp_req_chain *state = &req->chain;
200
201	state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
202					  CRYPTO_ACOMP_REQ_DST_VIRT);
203
204	if (acomp_request_src_isvirt(req)) {
205		unsigned int slen = req->slen;
206		const u8 *svirt = req->svirt;
207
208		state->src = svirt;
209		sg_init_one(&state->ssg, svirt, slen);
210		acomp_request_set_src_sg(req, &state->ssg, slen);
211	}
212
213	if (acomp_request_dst_isvirt(req)) {
214		unsigned int dlen = req->dlen;
215		u8 *dvirt = req->dvirt;
216
217		state->dst = dvirt;
218		sg_init_one(&state->dsg, dvirt, dlen);
219		acomp_request_set_dst_sg(req, &state->dsg, dlen);
220	}
221}
222
223static int acomp_do_nondma(struct acomp_req *req, bool comp)
224{
225	ACOMP_FBREQ_ON_STACK(fbreq, req);
226	int err;
227
228	if (comp)
229		err = crypto_acomp_compress(fbreq);
230	else
231		err = crypto_acomp_decompress(fbreq);
232
233	req->dlen = fbreq->dlen;
234	return err;
235}
236
237static int acomp_do_one_req(struct acomp_req *req, bool comp)
238{
239	if (acomp_request_isnondma(req))
240		return acomp_do_nondma(req, comp);
241
242	acomp_virt_to_sg(req);
243	return comp ? crypto_acomp_reqtfm(req)->compress(req) :
244		      crypto_acomp_reqtfm(req)->decompress(req);
245}
246
247static int acomp_reqchain_finish(struct acomp_req *req, int err)
248{
249	acomp_reqchain_virt(req);
250	acomp_restore_req(req);
251	return err;
252}
253
254static void acomp_reqchain_done(void *data, int err)
255{
256	struct acomp_req *req = data;
257	crypto_completion_t compl;
258
259	compl = req->chain.compl;
260	data = req->chain.data;
261
262	if (err == -EINPROGRESS)
263		goto notify;
264
265	err = acomp_reqchain_finish(req, err);
266
267notify:
268	compl(data, err);
269}
270
271static int acomp_do_req_chain(struct acomp_req *req, bool comp)
272{
273	int err;
274
275	acomp_save_req(req, acomp_reqchain_done);
276
277	err = acomp_do_one_req(req, comp);
278	if (err == -EBUSY || err == -EINPROGRESS)
279		return err;
280
281	return acomp_reqchain_finish(req, err);
282}
283
284int crypto_acomp_compress(struct acomp_req *req)
285{
286	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
287
288	if (acomp_req_on_stack(req) && acomp_is_async(tfm))
289		return -EAGAIN;
290	if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
291		return crypto_acomp_reqtfm(req)->compress(req);
292	return acomp_do_req_chain(req, true);
293}
294EXPORT_SYMBOL_GPL(crypto_acomp_compress);
295
296int crypto_acomp_decompress(struct acomp_req *req)
297{
298	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
299
300	if (acomp_req_on_stack(req) && acomp_is_async(tfm))
301		return -EAGAIN;
302	if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
303		return crypto_acomp_reqtfm(req)->decompress(req);
304	return acomp_do_req_chain(req, false);
305}
306EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
307
308void comp_prepare_alg(struct comp_alg_common *alg)
309{
310	struct crypto_alg *base = &alg->base;
311
312	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
313}
314
315int crypto_register_acomp(struct acomp_alg *alg)
316{
317	struct crypto_alg *base = &alg->calg.base;
318
319	comp_prepare_alg(&alg->calg);
320
321	base->cra_type = &crypto_acomp_type;
322	base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
323
324	return crypto_register_alg(base);
325}
326EXPORT_SYMBOL_GPL(crypto_register_acomp);
327
328void crypto_unregister_acomp(struct acomp_alg *alg)
329{
330	crypto_unregister_alg(&alg->base);
331}
332EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
333
334int crypto_register_acomps(struct acomp_alg *algs, int count)
335{
336	int i, ret;
337
338	for (i = 0; i < count; i++) {
339		ret = crypto_register_acomp(&algs[i]);
340		if (ret)
341			goto err;
342	}
343
344	return 0;
345
346err:
347	for (--i; i >= 0; --i)
348		crypto_unregister_acomp(&algs[i]);
349
350	return ret;
351}
352EXPORT_SYMBOL_GPL(crypto_register_acomps);
353
354void crypto_unregister_acomps(struct acomp_alg *algs, int count)
355{
356	int i;
357
358	for (i = count - 1; i >= 0; --i)
359		crypto_unregister_acomp(&algs[i]);
360}
361EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
362
363static void acomp_stream_workfn(struct work_struct *work)
364{
365	struct crypto_acomp_streams *s =
366		container_of(work, struct crypto_acomp_streams, stream_work);
367	struct crypto_acomp_stream __percpu *streams = s->streams;
368	int cpu;
369
370	for_each_cpu(cpu, &s->stream_want) {
371		struct crypto_acomp_stream *ps;
372		void *ctx;
373
374		ps = per_cpu_ptr(streams, cpu);
375		if (ps->ctx)
376			continue;
377
378		ctx = s->alloc_ctx();
379		if (IS_ERR(ctx))
380			break;
381
382		spin_lock_bh(&ps->lock);
383		ps->ctx = ctx;
384		spin_unlock_bh(&ps->lock);
385
386		cpumask_clear_cpu(cpu, &s->stream_want);
387	}
388}
389
390void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
391{
392	struct crypto_acomp_stream __percpu *streams = s->streams;
393	void (*free_ctx)(void *);
394	int i;
395
396	s->streams = NULL;
397	if (!streams)
398		return;
399
400	cancel_work_sync(&s->stream_work);
401	free_ctx = s->free_ctx;
402
403	for_each_possible_cpu(i) {
404		struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
405
406		if (!ps->ctx)
407			continue;
408
409		free_ctx(ps->ctx);
410	}
411
412	free_percpu(streams);
413}
414EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
415
416int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
417{
418	struct crypto_acomp_stream __percpu *streams;
419	struct crypto_acomp_stream *ps;
420	unsigned int i;
421	void *ctx;
422
423	if (s->streams)
424		return 0;
425
426	streams = alloc_percpu(struct crypto_acomp_stream);
427	if (!streams)
428		return -ENOMEM;
429
430	ctx = s->alloc_ctx();
431	if (IS_ERR(ctx)) {
432		free_percpu(streams);
433		return PTR_ERR(ctx);
434	}
435
436	i = cpumask_first(cpu_possible_mask);
437	ps = per_cpu_ptr(streams, i);
438	ps->ctx = ctx;
439
440	for_each_possible_cpu(i) {
441		ps = per_cpu_ptr(streams, i);
442		spin_lock_init(&ps->lock);
443	}
444
445	s->streams = streams;
446
447	INIT_WORK(&s->stream_work, acomp_stream_workfn);
448	return 0;
449}
450EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
451
452struct crypto_acomp_stream *crypto_acomp_lock_stream_bh(
453	struct crypto_acomp_streams *s) __acquires(stream)
454{
455	struct crypto_acomp_stream __percpu *streams = s->streams;
456	int cpu = raw_smp_processor_id();
457	struct crypto_acomp_stream *ps;
458
459	ps = per_cpu_ptr(streams, cpu);
460	spin_lock_bh(&ps->lock);
461	if (likely(ps->ctx))
462		return ps;
463	spin_unlock(&ps->lock);
464
465	cpumask_set_cpu(cpu, &s->stream_want);
466	schedule_work(&s->stream_work);
467
468	ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
469	spin_lock(&ps->lock);
470	return ps;
471}
472EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh);
473
474void acomp_walk_done_src(struct acomp_walk *walk, int used)
475{
476	walk->slen -= used;
477	if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
478		scatterwalk_advance(&walk->in, used);
479	else
480		scatterwalk_done_src(&walk->in, used);
481
482	if ((walk->flags & ACOMP_WALK_SLEEP))
483		cond_resched();
484}
485EXPORT_SYMBOL_GPL(acomp_walk_done_src);
486
487void acomp_walk_done_dst(struct acomp_walk *walk, int used)
488{
489	walk->dlen -= used;
490	if ((walk->flags & ACOMP_WALK_DST_LINEAR))
491		scatterwalk_advance(&walk->out, used);
492	else
493		scatterwalk_done_dst(&walk->out, used);
494
495	if ((walk->flags & ACOMP_WALK_SLEEP))
496		cond_resched();
497}
498EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
499
500int acomp_walk_next_src(struct acomp_walk *walk)
501{
502	unsigned int slen = walk->slen;
503	unsigned int max = UINT_MAX;
504
505	if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
506		max = PAGE_SIZE;
507	if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
508		walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
509					   walk->in.offset);
510		return min(slen, max);
511	}
512
513	return slen ? scatterwalk_next(&walk->in, slen) : 0;
514}
515EXPORT_SYMBOL_GPL(acomp_walk_next_src);
516
517int acomp_walk_next_dst(struct acomp_walk *walk)
518{
519	unsigned int dlen = walk->dlen;
520	unsigned int max = UINT_MAX;
521
522	if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
523		max = PAGE_SIZE;
524	if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
525		walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
526					    walk->out.offset);
527		return min(dlen, max);
528	}
529
530	return dlen ? scatterwalk_next(&walk->out, dlen) : 0;
531}
532EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
533
534int acomp_walk_virt(struct acomp_walk *__restrict walk,
535		    struct acomp_req *__restrict req, bool atomic)
536{
537	struct scatterlist *src = req->src;
538	struct scatterlist *dst = req->dst;
539
540	walk->slen = req->slen;
541	walk->dlen = req->dlen;
542
543	if (!walk->slen || !walk->dlen)
544		return -EINVAL;
545
546	walk->flags = 0;
547	if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
548		walk->flags |= ACOMP_WALK_SLEEP;
549	if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
550		walk->flags |= ACOMP_WALK_SRC_LINEAR;
551	if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
552		walk->flags |= ACOMP_WALK_DST_LINEAR;
553
554	if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
555		walk->in.sg = (void *)req->svirt;
556		walk->in.offset = 0;
557	} else
558		scatterwalk_start(&walk->in, src);
559	if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
560		walk->out.sg = (void *)req->dvirt;
561		walk->out.offset = 0;
562	} else
563		scatterwalk_start(&walk->out, dst);
564
565	return 0;
566}
567EXPORT_SYMBOL_GPL(acomp_walk_virt);
568
569struct acomp_req *acomp_request_clone(struct acomp_req *req,
570				      size_t total, gfp_t gfp)
571{
572	struct acomp_req *nreq;
573
574	nreq = container_of(crypto_request_clone(&req->base, total, gfp),
575			    struct acomp_req, base);
576	if (nreq == req)
577		return req;
578
579	if (req->src == &req->chain.ssg)
580		nreq->src = &nreq->chain.ssg;
581	if (req->dst == &req->chain.dsg)
582		nreq->dst = &nreq->chain.dsg;
583	return nreq;
584}
585EXPORT_SYMBOL_GPL(acomp_request_clone);
586
587MODULE_LICENSE("GPL");
588MODULE_DESCRIPTION("Asynchronous compression type");