backing_dev.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/blkdev.h>
  3
  4#include "../dm-core.h"
  5#include "pcache_internal.h"
  6#include "cache_dev.h"
  7#include "backing_dev.h"
  8#include "cache.h"
  9#include "dm_pcache.h"
 10
 11static struct kmem_cache *backing_req_cache;
 12static struct kmem_cache *backing_bvec_cache;
 13
 14static void backing_dev_exit(struct pcache_backing_dev *backing_dev)
 15{
 16	mempool_exit(&backing_dev->req_pool);
 17	mempool_exit(&backing_dev->bvec_pool);
 18}
 19
 20static void req_submit_fn(struct work_struct *work);
 21static void req_complete_fn(struct work_struct *work);
 22static int backing_dev_init(struct dm_pcache *pcache)
 23{
 24	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
 25	int ret;
 26
 27	ret = mempool_init_slab_pool(&backing_dev->req_pool, 128, backing_req_cache);
 28	if (ret)
 29		goto err;
 30
 31	ret = mempool_init_slab_pool(&backing_dev->bvec_pool, 128, backing_bvec_cache);
 32	if (ret)
 33		goto req_pool_exit;
 34
 35	INIT_LIST_HEAD(&backing_dev->submit_list);
 36	INIT_LIST_HEAD(&backing_dev->complete_list);
 37	spin_lock_init(&backing_dev->submit_lock);
 38	spin_lock_init(&backing_dev->complete_lock);
 39	INIT_WORK(&backing_dev->req_submit_work, req_submit_fn);
 40	INIT_WORK(&backing_dev->req_complete_work, req_complete_fn);
 41	atomic_set(&backing_dev->inflight_reqs, 0);
 42	init_waitqueue_head(&backing_dev->inflight_wq);
 43
 44	return 0;
 45
 46req_pool_exit:
 47	mempool_exit(&backing_dev->req_pool);
 48err:
 49	return ret;
 50}
 51
 52int backing_dev_start(struct dm_pcache *pcache)
 53{
 54	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
 55	int ret;
 56
 57	ret = backing_dev_init(pcache);
 58	if (ret)
 59		return ret;
 60
 61	backing_dev->dev_size = bdev_nr_sectors(backing_dev->dm_dev->bdev);
 62
 63	return 0;
 64}
 65
 66void backing_dev_stop(struct dm_pcache *pcache)
 67{
 68	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
 69
 70	/*
 71	 * There should not be any new request comming, just wait
 72	 * inflight requests done.
 73	 */
 74	wait_event(backing_dev->inflight_wq,
 75			atomic_read(&backing_dev->inflight_reqs) == 0);
 76
 77	flush_work(&backing_dev->req_submit_work);
 78	flush_work(&backing_dev->req_complete_work);
 79
 80	backing_dev_exit(backing_dev);
 81}
 82
 83/* pcache_backing_dev_req functions */
 84void backing_dev_req_end(struct pcache_backing_dev_req *backing_req)
 85{
 86	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
 87
 88	if (backing_req->end_req)
 89		backing_req->end_req(backing_req, backing_req->ret);
 90
 91	switch (backing_req->type) {
 92	case BACKING_DEV_REQ_TYPE_REQ:
 93		if (backing_req->req.upper_req)
 94			pcache_req_put(backing_req->req.upper_req, backing_req->ret);
 95		break;
 96	case BACKING_DEV_REQ_TYPE_KMEM:
 97		if (backing_req->kmem.bvecs != backing_req->kmem.inline_bvecs)
 98			mempool_free(backing_req->kmem.bvecs, &backing_dev->bvec_pool);
 99		break;
100	default:
101		BUG();
102	}
103
104	mempool_free(backing_req, &backing_dev->req_pool);
105
106	if (atomic_dec_and_test(&backing_dev->inflight_reqs))
107		wake_up(&backing_dev->inflight_wq);
108}
109
110static void req_complete_fn(struct work_struct *work)
111{
112	struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_complete_work);
113	struct pcache_backing_dev_req *backing_req;
114	LIST_HEAD(tmp_list);
115
116	spin_lock_irq(&backing_dev->complete_lock);
117	list_splice_init(&backing_dev->complete_list, &tmp_list);
118	spin_unlock_irq(&backing_dev->complete_lock);
119
120	while (!list_empty(&tmp_list)) {
121		backing_req = list_first_entry(&tmp_list,
122					    struct pcache_backing_dev_req, node);
123		list_del_init(&backing_req->node);
124		backing_dev_req_end(backing_req);
125	}
126}
127
128static void backing_dev_bio_end(struct bio *bio)
129{
130	struct pcache_backing_dev_req *backing_req = bio->bi_private;
131	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
132	unsigned long flags;
133
134	backing_req->ret = blk_status_to_errno(bio->bi_status);
135
136	spin_lock_irqsave(&backing_dev->complete_lock, flags);
137	list_move_tail(&backing_req->node, &backing_dev->complete_list);
138	queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_complete_work);
139	spin_unlock_irqrestore(&backing_dev->complete_lock, flags);
140}
141
142static void req_submit_fn(struct work_struct *work)
143{
144	struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_submit_work);
145	struct pcache_backing_dev_req *backing_req;
146	LIST_HEAD(tmp_list);
147
148	spin_lock(&backing_dev->submit_lock);
149	list_splice_init(&backing_dev->submit_list, &tmp_list);
150	spin_unlock(&backing_dev->submit_lock);
151
152	while (!list_empty(&tmp_list)) {
153		backing_req = list_first_entry(&tmp_list,
154					    struct pcache_backing_dev_req, node);
155		list_del_init(&backing_req->node);
156		submit_bio_noacct(&backing_req->bio);
157	}
158}
159
160void backing_dev_req_submit(struct pcache_backing_dev_req *backing_req, bool direct)
161{
162	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
163
164	if (direct) {
165		submit_bio_noacct(&backing_req->bio);
166		return;
167	}
168
169	spin_lock(&backing_dev->submit_lock);
170	list_add_tail(&backing_req->node, &backing_dev->submit_list);
171	queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_submit_work);
172	spin_unlock(&backing_dev->submit_lock);
173}
174
175static void bio_map(struct bio *bio, void *base, size_t size)
176{
177	struct page *page;
178	unsigned int offset;
179	unsigned int len;
180
181	if (!is_vmalloc_addr(base)) {
182		page = virt_to_page(base);
183		offset = offset_in_page(base);
184
185		BUG_ON(!bio_add_page(bio, page, size, offset));
186		return;
187	}
188
189	flush_kernel_vmap_range(base, size);
190	while (size) {
191		page = vmalloc_to_page(base);
192		offset = offset_in_page(base);
193		len = min_t(size_t, PAGE_SIZE - offset, size);
194
195		BUG_ON(!bio_add_page(bio, page, len, offset));
196		size -= len;
197		base += len;
198	}
199}
200
201static struct pcache_backing_dev_req *req_type_req_alloc(struct pcache_backing_dev *backing_dev,
202							struct pcache_backing_dev_req_opts *opts)
203{
204	struct pcache_request *pcache_req = opts->req.upper_req;
205	struct pcache_backing_dev_req *backing_req;
206	struct bio *orig = pcache_req->bio;
207
208	backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
209	if (!backing_req)
210		return NULL;
211
212	memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));
213
214	bio_init_clone(backing_dev->dm_dev->bdev, &backing_req->bio, orig, opts->gfp_mask);
215
216	backing_req->type = BACKING_DEV_REQ_TYPE_REQ;
217	backing_req->backing_dev = backing_dev;
218	atomic_inc(&backing_dev->inflight_reqs);
219
220	return backing_req;
221}
222
223static struct pcache_backing_dev_req *kmem_type_req_alloc(struct pcache_backing_dev *backing_dev,
224						struct pcache_backing_dev_req_opts *opts)
225{
226	struct pcache_backing_dev_req *backing_req;
227	u32 n_vecs = bio_add_max_vecs(opts->kmem.data, opts->kmem.len);
228
229	backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
230	if (!backing_req)
231		return NULL;
232
233	memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));
234
235	if (n_vecs > BACKING_DEV_REQ_INLINE_BVECS) {
236		backing_req->kmem.bvecs = mempool_alloc(&backing_dev->bvec_pool, opts->gfp_mask);
237		if (!backing_req->kmem.bvecs)
238			goto free_backing_req;
239	} else {
240		backing_req->kmem.bvecs = backing_req->kmem.inline_bvecs;
241	}
242
243	backing_req->kmem.n_vecs = n_vecs;
244	backing_req->type = BACKING_DEV_REQ_TYPE_KMEM;
245	backing_req->backing_dev = backing_dev;
246	atomic_inc(&backing_dev->inflight_reqs);
247
248	return backing_req;
249
250free_backing_req:
251	mempool_free(backing_req, &backing_dev->req_pool);
252	return NULL;
253}
254
255struct pcache_backing_dev_req *backing_dev_req_alloc(struct pcache_backing_dev *backing_dev,
256						struct pcache_backing_dev_req_opts *opts)
257{
258	if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
259		return req_type_req_alloc(backing_dev, opts);
260
261	if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
262		return kmem_type_req_alloc(backing_dev, opts);
263
264	BUG();
265}
266
267static void req_type_req_init(struct pcache_backing_dev_req *backing_req,
268			struct pcache_backing_dev_req_opts *opts)
269{
270	struct pcache_request *pcache_req = opts->req.upper_req;
271	struct bio *clone;
272	u32 off = opts->req.req_off;
273	u32 len = opts->req.len;
274
275	clone = &backing_req->bio;
276	BUG_ON(off & SECTOR_MASK);
277	BUG_ON(len & SECTOR_MASK);
278	bio_trim(clone, off >> SECTOR_SHIFT, len >> SECTOR_SHIFT);
279
280	clone->bi_iter.bi_sector = (pcache_req->off + off) >> SECTOR_SHIFT;
281	clone->bi_private = backing_req;
282	clone->bi_end_io = backing_dev_bio_end;
283
284	INIT_LIST_HEAD(&backing_req->node);
285	backing_req->end_req     = opts->end_fn;
286
287	pcache_req_get(pcache_req);
288	backing_req->req.upper_req	= pcache_req;
289	backing_req->req.bio_off	= off;
290}
291
292static void kmem_type_req_init(struct pcache_backing_dev_req *backing_req,
293			struct pcache_backing_dev_req_opts *opts)
294{
295	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
296	struct bio *backing_bio;
297
298	bio_init(&backing_req->bio, backing_dev->dm_dev->bdev, backing_req->kmem.bvecs,
299			backing_req->kmem.n_vecs, opts->kmem.opf);
300
301	backing_bio = &backing_req->bio;
302	bio_map(backing_bio, opts->kmem.data, opts->kmem.len);
303
304	backing_bio->bi_iter.bi_sector = (opts->kmem.backing_off) >> SECTOR_SHIFT;
305	backing_bio->bi_private = backing_req;
306	backing_bio->bi_end_io = backing_dev_bio_end;
307
308	INIT_LIST_HEAD(&backing_req->node);
309	backing_req->end_req	= opts->end_fn;
310	backing_req->priv_data	= opts->priv_data;
311}
312
313void backing_dev_req_init(struct pcache_backing_dev_req *backing_req,
314			struct pcache_backing_dev_req_opts *opts)
315{
316	if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
317		return req_type_req_init(backing_req, opts);
318
319	if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
320		return kmem_type_req_init(backing_req, opts);
321
322	BUG();
323}
324
325struct pcache_backing_dev_req *backing_dev_req_create(struct pcache_backing_dev *backing_dev,
326						struct pcache_backing_dev_req_opts *opts)
327{
328	struct pcache_backing_dev_req *backing_req;
329
330	backing_req = backing_dev_req_alloc(backing_dev, opts);
331	if (!backing_req)
332		return NULL;
333
334	backing_dev_req_init(backing_req, opts);
335
336	return backing_req;
337}
338
339void backing_dev_flush(struct pcache_backing_dev *backing_dev)
340{
341	blkdev_issue_flush(backing_dev->dm_dev->bdev);
342}
343
344int pcache_backing_init(void)
345{
346	u32 max_bvecs = (PCACHE_CACHE_SUBTREE_SIZE >> PAGE_SHIFT) + 1;
347	int ret;
348
349	backing_req_cache = KMEM_CACHE(pcache_backing_dev_req, 0);
350	if (!backing_req_cache) {
351		ret = -ENOMEM;
352		goto err;
353	}
354
355	backing_bvec_cache = kmem_cache_create("pcache-bvec-slab",
356					max_bvecs * sizeof(struct bio_vec),
357					0, 0, NULL);
358	if (!backing_bvec_cache) {
359		ret = -ENOMEM;
360		goto destroy_req_cache;
361	}
362
363	return 0;
364destroy_req_cache:
365	kmem_cache_destroy(backing_req_cache);
366err:
367	return ret;
368}
369
370void pcache_backing_exit(void)
371{
372	kmem_cache_destroy(backing_bvec_cache);
373	kmem_cache_destroy(backing_req_cache);
374}