decompressor_zstd.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/zstd.h>
  3#include "compress.h"
  4
  5struct z_erofs_zstd {
  6	struct z_erofs_zstd *next;
  7	u8 bounce[PAGE_SIZE];
  8	void *wksp;
  9	unsigned int wkspsz;
 10};
 11
 12static DEFINE_SPINLOCK(z_erofs_zstd_lock);
 13static unsigned int z_erofs_zstd_max_dictsize;
 14static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
 15static struct z_erofs_zstd *z_erofs_zstd_head;
 16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
 17
 18module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
 19
 20static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
 21{
 22	struct z_erofs_zstd *strm;
 23
 24again:
 25	spin_lock(&z_erofs_zstd_lock);
 26	strm = z_erofs_zstd_head;
 27	if (!strm) {
 28		spin_unlock(&z_erofs_zstd_lock);
 29		wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
 30		goto again;
 31	}
 32	z_erofs_zstd_head = all ? NULL : strm->next;
 33	spin_unlock(&z_erofs_zstd_lock);
 34	return strm;
 35}
 36
 37static void z_erofs_zstd_exit(void)
 38{
 39	while (z_erofs_zstd_avail_strms) {
 40		struct z_erofs_zstd *strm, *n;
 41
 42		for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
 43			n = strm->next;
 44
 45			kvfree(strm->wksp);
 46			kfree(strm);
 47			--z_erofs_zstd_avail_strms;
 48		}
 49	}
 50}
 51
 52static int __init z_erofs_zstd_init(void)
 53{
 54	/* by default, use # of possible CPUs instead */
 55	if (!z_erofs_zstd_nstrms)
 56		z_erofs_zstd_nstrms = num_possible_cpus();
 57
 58	for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
 59	     ++z_erofs_zstd_avail_strms) {
 60		struct z_erofs_zstd *strm;
 61
 62		strm = kzalloc(sizeof(*strm), GFP_KERNEL);
 63		if (!strm) {
 64			z_erofs_zstd_exit();
 65			return -ENOMEM;
 66		}
 67		spin_lock(&z_erofs_zstd_lock);
 68		strm->next = z_erofs_zstd_head;
 69		z_erofs_zstd_head = strm;
 70		spin_unlock(&z_erofs_zstd_lock);
 71	}
 72	return 0;
 73}
 74
 75static int z_erofs_load_zstd_config(struct super_block *sb,
 76			struct erofs_super_block *dsb, void *data, int size)
 77{
 78	static DEFINE_MUTEX(zstd_resize_mutex);
 79	struct z_erofs_zstd_cfgs *zstd = data;
 80	unsigned int dict_size, wkspsz;
 81	struct z_erofs_zstd *strm, *head = NULL;
 82	void *wksp;
 83
 84	if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
 85		erofs_err(sb, "unsupported zstd format, size=%u", size);
 86		return -EINVAL;
 87	}
 88
 89	if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
 90		erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
 91		return -EINVAL;
 92	}
 93	dict_size = 1U << (zstd->windowlog + 10);
 94
 95	/* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
 96	mutex_lock(&zstd_resize_mutex);
 97	if (z_erofs_zstd_max_dictsize >= dict_size) {
 98		mutex_unlock(&zstd_resize_mutex);
 99		return 0;
100	}
101
102	/* 1. collect/isolate all streams for the following check */
103	while (z_erofs_zstd_avail_strms) {
104		struct z_erofs_zstd *n;
105
106		for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
107			n = strm->next;
108			strm->next = head;
109			head = strm;
110			--z_erofs_zstd_avail_strms;
111		}
112	}
113
114	/* 2. walk each isolated stream and grow max dict_size if needed */
115	wkspsz = zstd_dstream_workspace_bound(dict_size);
116	for (strm = head; strm; strm = strm->next) {
117		wksp = kvmalloc(wkspsz, GFP_KERNEL);
118		if (!wksp)
119			break;
120		kvfree(strm->wksp);
121		strm->wksp = wksp;
122		strm->wkspsz = wkspsz;
123	}
124
125	/* 3. push back all to the global list and update max dict_size */
126	spin_lock(&z_erofs_zstd_lock);
127	DBG_BUGON(z_erofs_zstd_head);
128	z_erofs_zstd_head = head;
129	spin_unlock(&z_erofs_zstd_lock);
130	z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
131	wake_up_all(&z_erofs_zstd_wq);
132	if (!strm)
133		z_erofs_zstd_max_dictsize = dict_size;
134	mutex_unlock(&zstd_resize_mutex);
135	return strm ? -ENOMEM : 0;
136}
137
138static const char *z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
139					   struct page **pgpl)
140{
141	struct super_block *sb = rq->sb;
142	struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
143	zstd_in_buffer in_buf = { NULL, 0, 0 };
144	zstd_out_buffer out_buf = { NULL, 0, 0 };
145	struct z_erofs_zstd *strm;
146	zstd_dstream *stream;
147	const char *reason;
148	int zerr;
149
150	/* 1. get the exact compressed size */
151	dctx.kin = kmap_local_page(*rq->in);
152	reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
153			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
154	if (reason) {
155		kunmap_local(dctx.kin);
156		return reason;
157	}
158
159	/* 2. get an available ZSTD context */
160	strm = z_erofs_isolate_strms(false);
161
162	/* 3. multi-call decompress */
163	stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
164	if (!stream) {
165		reason = ERR_PTR(-ENOMEM);
166		goto failed_zinit;
167	}
168
169	rq->fillgaps = true;	/* ZSTD doesn't support NULL output buffer */
170	in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
171	rq->inputsize -= in_buf.size;
172	in_buf.src = dctx.kin + rq->pageofs_in;
173	dctx.bounce = strm->bounce;
174
175	do {
176		dctx.inbuf_sz = in_buf.size;
177		dctx.inbuf_pos = in_buf.pos;
178		reason = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
179						 (void **)&in_buf.src, pgpl);
180		if (reason)
181			break;
182
183		if (out_buf.size == out_buf.pos) {
184			out_buf.size = dctx.avail_out;
185			out_buf.pos = 0;
186		}
187		in_buf.size = dctx.inbuf_sz;
188		in_buf.pos = dctx.inbuf_pos;
189
190		zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
191		dctx.avail_out = out_buf.size - out_buf.pos;
192		if (zstd_is_error(zerr) ||
193		    ((rq->outputsize + dctx.avail_out) && (!zerr || (zerr > 0 &&
194				!(rq->inputsize + in_buf.size - in_buf.pos))))) {
195			reason = zstd_is_error(zerr) ? zstd_get_error_name(zerr) :
196					"unexpected end of stream";
197			break;
198		}
199	} while (rq->outputsize + dctx.avail_out);
200
201	if (dctx.kout)
202		kunmap_local(dctx.kout);
203failed_zinit:
204	kunmap_local(dctx.kin);
205	/* 4. push back ZSTD stream context to the global list */
206	spin_lock(&z_erofs_zstd_lock);
207	strm->next = z_erofs_zstd_head;
208	z_erofs_zstd_head = strm;
209	spin_unlock(&z_erofs_zstd_lock);
210	wake_up(&z_erofs_zstd_wq);
211	return reason;
212}
213
214const struct z_erofs_decompressor z_erofs_zstd_decomp = {
215	.config = z_erofs_load_zstd_config,
216	.decompress = z_erofs_zstd_decompress,
217	.init = z_erofs_zstd_init,
218	.exit = z_erofs_zstd_exit,
219	.name = "zstd",
220};