decompressor_lzma.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/xz.h>
  3#include "compress.h"
  4
  5struct z_erofs_lzma {
  6	struct z_erofs_lzma *next;
  7	struct xz_dec_microlzma *state;
  8	u8 bounce[PAGE_SIZE];
  9};
 10
 11/* considering the LZMA performance, no need to use a lockless list for now */
 12static DEFINE_SPINLOCK(z_erofs_lzma_lock);
 13static unsigned int z_erofs_lzma_max_dictsize;
 14static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
 15static struct z_erofs_lzma *z_erofs_lzma_head;
 16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
 17
 18module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
 19
 20static void z_erofs_lzma_exit(void)
 21{
 22	/* there should be no running fs instance */
 23	while (z_erofs_lzma_avail_strms) {
 24		struct z_erofs_lzma *strm;
 25
 26		spin_lock(&z_erofs_lzma_lock);
 27		strm = z_erofs_lzma_head;
 28		if (!strm) {
 29			spin_unlock(&z_erofs_lzma_lock);
 30			DBG_BUGON(1);
 31			return;
 32		}
 33		z_erofs_lzma_head = NULL;
 34		spin_unlock(&z_erofs_lzma_lock);
 35
 36		while (strm) {
 37			struct z_erofs_lzma *n = strm->next;
 38
 39			if (strm->state)
 40				xz_dec_microlzma_end(strm->state);
 41			kfree(strm);
 42			--z_erofs_lzma_avail_strms;
 43			strm = n;
 44		}
 45	}
 46}
 47
 48static int __init z_erofs_lzma_init(void)
 49{
 50	unsigned int i;
 51
 52	/* by default, use # of possible CPUs instead */
 53	if (!z_erofs_lzma_nstrms)
 54		z_erofs_lzma_nstrms = num_possible_cpus();
 55
 56	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
 57		struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
 58
 59		if (!strm) {
 60			z_erofs_lzma_exit();
 61			return -ENOMEM;
 62		}
 63		spin_lock(&z_erofs_lzma_lock);
 64		strm->next = z_erofs_lzma_head;
 65		z_erofs_lzma_head = strm;
 66		spin_unlock(&z_erofs_lzma_lock);
 67		++z_erofs_lzma_avail_strms;
 68	}
 69	return 0;
 70}
 71
 72static int z_erofs_load_lzma_config(struct super_block *sb,
 73			struct erofs_super_block *dsb, void *data, int size)
 74{
 75	static DEFINE_MUTEX(lzma_resize_mutex);
 76	struct z_erofs_lzma_cfgs *lzma = data;
 77	unsigned int dict_size, i;
 78	struct z_erofs_lzma *strm, *head = NULL;
 79	int err;
 80
 81	if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
 82		erofs_err(sb, "invalid lzma cfgs, size=%u", size);
 83		return -EINVAL;
 84	}
 85	if (lzma->format) {
 86		erofs_err(sb, "unidentified lzma format %x, please check kernel version",
 87			  le16_to_cpu(lzma->format));
 88		return -EINVAL;
 89	}
 90	dict_size = le32_to_cpu(lzma->dict_size);
 91	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
 92		erofs_err(sb, "unsupported lzma dictionary size %u",
 93			  dict_size);
 94		return -EINVAL;
 95	}
 96
 97	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
 98	mutex_lock(&lzma_resize_mutex);
 99
100	if (z_erofs_lzma_max_dictsize >= dict_size) {
101		mutex_unlock(&lzma_resize_mutex);
102		return 0;
103	}
104
105	/* 1. collect/isolate all streams for the following check */
106	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
107		struct z_erofs_lzma *last;
108
109again:
110		spin_lock(&z_erofs_lzma_lock);
111		strm = z_erofs_lzma_head;
112		if (!strm) {
113			spin_unlock(&z_erofs_lzma_lock);
114			wait_event(z_erofs_lzma_wq,
115				   READ_ONCE(z_erofs_lzma_head));
116			goto again;
117		}
118		z_erofs_lzma_head = NULL;
119		spin_unlock(&z_erofs_lzma_lock);
120
121		for (last = strm; last->next; last = last->next)
122			++i;
123		last->next = head;
124		head = strm;
125	}
126
127	err = 0;
128	/* 2. walk each isolated stream and grow max dict_size if needed */
129	for (strm = head; strm; strm = strm->next) {
130		if (strm->state)
131			xz_dec_microlzma_end(strm->state);
132		strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
133		if (!strm->state)
134			err = -ENOMEM;
135	}
136
137	/* 3. push back all to the global list and update max dict_size */
138	spin_lock(&z_erofs_lzma_lock);
139	DBG_BUGON(z_erofs_lzma_head);
140	z_erofs_lzma_head = head;
141	spin_unlock(&z_erofs_lzma_lock);
142	wake_up_all(&z_erofs_lzma_wq);
143
144	z_erofs_lzma_max_dictsize = dict_size;
145	mutex_unlock(&lzma_resize_mutex);
146	return err;
147}
148
149static const char *z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
150					   struct page **pgpl)
151{
152	struct super_block *sb = rq->sb;
153	struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
154	struct xz_buf buf = {};
155	struct z_erofs_lzma *strm;
156	enum xz_ret xz_err;
157	const char *reason;
158
159	/* 1. get the exact LZMA compressed size */
160	dctx.kin = kmap_local_page(*rq->in);
161	reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
162			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
163	if (reason) {
164		kunmap_local(dctx.kin);
165		return reason;
166	}
167
168	/* 2. get an available lzma context */
169again:
170	spin_lock(&z_erofs_lzma_lock);
171	strm = z_erofs_lzma_head;
172	if (!strm) {
173		spin_unlock(&z_erofs_lzma_lock);
174		wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
175		goto again;
176	}
177	z_erofs_lzma_head = strm->next;
178	spin_unlock(&z_erofs_lzma_lock);
179
180	/* 3. multi-call decompress */
181	xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
182			       !rq->partial_decoding);
183	buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
184	rq->inputsize -= buf.in_size;
185	buf.in = dctx.kin + rq->pageofs_in;
186	dctx.bounce = strm->bounce;
187	do {
188		dctx.avail_out = buf.out_size - buf.out_pos;
189		dctx.inbuf_sz = buf.in_size;
190		dctx.inbuf_pos = buf.in_pos;
191		reason = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
192						    (void **)&buf.in, pgpl);
193		if (reason)
194			break;
195
196		if (buf.out_size == buf.out_pos) {
197			buf.out_size = dctx.avail_out;
198			buf.out_pos = 0;
199		}
200		buf.in_size = dctx.inbuf_sz;
201		buf.in_pos = dctx.inbuf_pos;
202
203		xz_err = xz_dec_microlzma_run(strm->state, &buf);
204		DBG_BUGON(buf.out_pos > buf.out_size);
205		DBG_BUGON(buf.in_pos > buf.in_size);
206
207		if (xz_err != XZ_OK) {
208			if (xz_err == XZ_STREAM_END && !rq->outputsize)
209				break;
210			reason = (xz_err == XZ_DATA_ERROR ?
211				"corrupted compressed data" :
212				"unexpected end of stream");
213			break;
214		}
215	} while (1);
216
217	if (dctx.kout)
218		kunmap_local(dctx.kout);
219	kunmap_local(dctx.kin);
220	/* 4. push back LZMA stream context to the global list */
221	spin_lock(&z_erofs_lzma_lock);
222	strm->next = z_erofs_lzma_head;
223	z_erofs_lzma_head = strm;
224	spin_unlock(&z_erofs_lzma_lock);
225	wake_up(&z_erofs_lzma_wq);
226	return reason;
227}
228
229const struct z_erofs_decompressor z_erofs_lzma_decomp = {
230	.config = z_erofs_load_lzma_config,
231	.decompress = z_erofs_lzma_decompress,
232	.init = z_erofs_lzma_init,
233	.exit = z_erofs_lzma_exit,
234	.name = "lzma"
235};