cache.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/blk_types.h>
  3
  4#include "cache.h"
  5#include "cache_dev.h"
  6#include "backing_dev.h"
  7#include "dm_pcache.h"
  8
  9struct kmem_cache *key_cache;
 10
 11static inline struct pcache_cache_info *get_cache_info_addr(struct pcache_cache *cache)
 12{
 13	return (struct pcache_cache_info *)((char *)cache->cache_info_addr +
 14						(size_t)cache->info_index * PCACHE_CACHE_INFO_SIZE);
 15}
 16
 17static void cache_info_write(struct pcache_cache *cache)
 18{
 19	struct pcache_cache_info *cache_info = &cache->cache_info;
 20
 21	cache_info->header.seq++;
 22	cache_info->header.crc = pcache_meta_crc(&cache_info->header,
 23						sizeof(struct pcache_cache_info));
 24
 25	cache->info_index = (cache->info_index + 1) % PCACHE_META_INDEX_MAX;
 26	memcpy_flushcache(get_cache_info_addr(cache), cache_info,
 27			sizeof(struct pcache_cache_info));
 28	pmem_wmb();
 29}
 30
 31static void cache_info_init_default(struct pcache_cache *cache);
 32static int cache_info_init(struct pcache_cache *cache, struct pcache_cache_options *opts)
 33{
 34	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
 35	struct pcache_cache_info *cache_info_addr;
 36
 37	cache_info_addr = pcache_meta_find_latest(&cache->cache_info_addr->header,
 38						sizeof(struct pcache_cache_info),
 39						PCACHE_CACHE_INFO_SIZE,
 40						&cache->cache_info);
 41	if (IS_ERR(cache_info_addr))
 42		return PTR_ERR(cache_info_addr);
 43
 44	if (cache_info_addr) {
 45		if (opts->data_crc !=
 46				(cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC)) {
 47			pcache_dev_err(pcache, "invalid option for data_crc: %s, expected: %s",
 48					opts->data_crc ? "true" : "false",
 49					cache->cache_info.flags & PCACHE_CACHE_FLAGS_DATA_CRC ? "true" : "false");
 50			return -EINVAL;
 51		}
 52
 53		cache->info_index = ((char *)cache_info_addr - (char *)cache->cache_info_addr) / PCACHE_CACHE_INFO_SIZE;
 54
 55		return 0;
 56	}
 57
 58	/* init cache_info for new cache */
 59	cache_info_init_default(cache);
 60	cache_mode_set(cache, opts->cache_mode);
 61	if (opts->data_crc)
 62		cache->cache_info.flags |= PCACHE_CACHE_FLAGS_DATA_CRC;
 63
 64	return 0;
 65}
 66
 67static void cache_info_set_gc_percent(struct pcache_cache_info *cache_info, u8 percent)
 68{
 69	cache_info->flags &= ~PCACHE_CACHE_FLAGS_GC_PERCENT_MASK;
 70	cache_info->flags |= FIELD_PREP(PCACHE_CACHE_FLAGS_GC_PERCENT_MASK, percent);
 71}
 72
 73int pcache_cache_set_gc_percent(struct pcache_cache *cache, u8 percent)
 74{
 75	if (percent > PCACHE_CACHE_GC_PERCENT_MAX || percent < PCACHE_CACHE_GC_PERCENT_MIN)
 76		return -EINVAL;
 77
 78	mutex_lock(&cache->cache_info_lock);
 79	cache_info_set_gc_percent(&cache->cache_info, percent);
 80
 81	cache_info_write(cache);
 82	mutex_unlock(&cache->cache_info_lock);
 83
 84	return 0;
 85}
 86
 87void cache_pos_encode(struct pcache_cache *cache,
 88			     struct pcache_cache_pos_onmedia *pos_onmedia_base,
 89			     struct pcache_cache_pos *pos, u64 seq, u32 *index)
 90{
 91	struct pcache_cache_pos_onmedia pos_onmedia;
 92	struct pcache_cache_pos_onmedia *pos_onmedia_addr = pos_onmedia_base + *index;
 93
 94	pos_onmedia.cache_seg_id = pos->cache_seg->cache_seg_id;
 95	pos_onmedia.seg_off = pos->seg_off;
 96	pos_onmedia.header.seq = seq;
 97	pos_onmedia.header.crc = cache_pos_onmedia_crc(&pos_onmedia);
 98
 99	*index = (*index + 1) % PCACHE_META_INDEX_MAX;
100
101	memcpy_flushcache(pos_onmedia_addr, &pos_onmedia, sizeof(struct pcache_cache_pos_onmedia));
102	pmem_wmb();
103}
104
105int cache_pos_decode(struct pcache_cache *cache,
106			    struct pcache_cache_pos_onmedia *pos_onmedia,
107			    struct pcache_cache_pos *pos, u64 *seq, u32 *index)
108{
109	struct pcache_cache_pos_onmedia latest, *latest_addr;
110
111	latest_addr = pcache_meta_find_latest(&pos_onmedia->header,
112					sizeof(struct pcache_cache_pos_onmedia),
113					sizeof(struct pcache_cache_pos_onmedia),
114					&latest);
115	if (IS_ERR(latest_addr))
116		return PTR_ERR(latest_addr);
117
118	if (!latest_addr)
119		return -EIO;
120
121	pos->cache_seg = &cache->segments[latest.cache_seg_id];
122	pos->seg_off = latest.seg_off;
123	*seq = latest.header.seq;
124	*index = (latest_addr - pos_onmedia);
125
126	return 0;
127}
128
129static inline void cache_info_set_seg_id(struct pcache_cache *cache, u32 seg_id)
130{
131	cache->cache_info.seg_id = seg_id;
132}
133
134static int cache_init(struct dm_pcache *pcache)
135{
136	struct pcache_cache *cache = &pcache->cache;
137	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
138	struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
139	int ret;
140
141	cache->segments = kvcalloc(cache_dev->seg_num, sizeof(struct pcache_cache_segment), GFP_KERNEL);
142	if (!cache->segments) {
143		ret = -ENOMEM;
144		goto err;
145	}
146
147	cache->seg_map = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);
148	if (!cache->seg_map) {
149		ret = -ENOMEM;
150		goto free_segments;
151	}
152
153	cache->backing_dev = backing_dev;
154	cache->cache_dev = &pcache->cache_dev;
155	cache->n_segs = cache_dev->seg_num;
156	atomic_set(&cache->gc_errors, 0);
157	spin_lock_init(&cache->seg_map_lock);
158	spin_lock_init(&cache->key_head_lock);
159
160	mutex_init(&cache->cache_info_lock);
161	mutex_init(&cache->key_tail_lock);
162	mutex_init(&cache->dirty_tail_lock);
163	mutex_init(&cache->writeback_lock);
164
165	INIT_DELAYED_WORK(&cache->writeback_work, cache_writeback_fn);
166	INIT_DELAYED_WORK(&cache->gc_work, pcache_cache_gc_fn);
167	INIT_WORK(&cache->clean_work, clean_fn);
168
169	return 0;
170
171free_segments:
172	kvfree(cache->segments);
173err:
174	return ret;
175}
176
177static void cache_exit(struct pcache_cache *cache)
178{
179	kvfree(cache->seg_map);
180	kvfree(cache->segments);
181}
182
183static void cache_info_init_default(struct pcache_cache *cache)
184{
185	struct pcache_cache_info *cache_info = &cache->cache_info;
186
187	memset(cache_info, 0, sizeof(*cache_info));
188	cache_info->n_segs = cache->cache_dev->seg_num;
189	cache_info_set_gc_percent(cache_info, PCACHE_CACHE_GC_PERCENT_DEFAULT);
190}
191
192static int cache_tail_init(struct pcache_cache *cache)
193{
194	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
195	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
196
197	if (new_cache) {
198		__set_bit(0, cache->seg_map);
199
200		cache->key_head.cache_seg = &cache->segments[0];
201		cache->key_head.seg_off = 0;
202		cache_pos_copy(&cache->key_tail, &cache->key_head);
203		cache_pos_copy(&cache->dirty_tail, &cache->key_head);
204
205		cache_encode_dirty_tail(cache);
206		cache_encode_key_tail(cache);
207	} else {
208		if (cache_decode_key_tail(cache) || cache_decode_dirty_tail(cache)) {
209			pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n");
210			return -EIO;
211		}
212	}
213
214	return 0;
215}
216
217static int get_seg_id(struct pcache_cache *cache,
218		      struct pcache_cache_segment *prev_cache_seg,
219		      bool new_cache, u32 *seg_id)
220{
221	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
222	struct pcache_cache_dev *cache_dev = cache->cache_dev;
223	int ret;
224
225	if (new_cache) {
226		ret = cache_dev_get_empty_segment_id(cache_dev, seg_id);
227		if (ret) {
228			pcache_dev_err(pcache, "no available segment\n");
229			goto err;
230		}
231
232		if (prev_cache_seg)
233			cache_seg_set_next_seg(prev_cache_seg, *seg_id);
234		else
235			cache_info_set_seg_id(cache, *seg_id);
236	} else {
237		if (prev_cache_seg) {
238			struct pcache_segment_info *prev_seg_info;
239
240			prev_seg_info = &prev_cache_seg->cache_seg_info;
241			if (!segment_info_has_next(prev_seg_info)) {
242				ret = -EFAULT;
243				goto err;
244			}
245			*seg_id = prev_cache_seg->cache_seg_info.next_seg;
246		} else {
247			*seg_id = cache->cache_info.seg_id;
248		}
249	}
250	return 0;
251err:
252	return ret;
253}
254
255static int cache_segs_init(struct pcache_cache *cache)
256{
257	struct pcache_cache_segment *prev_cache_seg = NULL;
258	struct pcache_cache_info *cache_info = &cache->cache_info;
259	bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE);
260	u32 seg_id;
261	int ret;
262	u32 i;
263
264	for (i = 0; i < cache_info->n_segs; i++) {
265		ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id);
266		if (ret)
267			goto err;
268
269		ret = cache_seg_init(cache, seg_id, i, new_cache);
270		if (ret)
271			goto err;
272
273		prev_cache_seg = &cache->segments[i];
274	}
275	return 0;
276err:
277	return ret;
278}
279
280static int cache_init_req_keys(struct pcache_cache *cache, u32 n_paral)
281{
282	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
283	u32 n_subtrees;
284	int ret;
285	u32 i, cpu;
286
287	/* Calculate number of cache trees based on the device size */
288	n_subtrees = DIV_ROUND_UP(cache->dev_size << SECTOR_SHIFT, PCACHE_CACHE_SUBTREE_SIZE);
289	ret = cache_tree_init(cache, &cache->req_key_tree, n_subtrees);
290	if (ret)
291		goto err;
292
293	cache->n_ksets = n_paral;
294	cache->ksets = kvcalloc(cache->n_ksets, PCACHE_KSET_SIZE, GFP_KERNEL);
295	if (!cache->ksets) {
296		ret = -ENOMEM;
297		goto req_tree_exit;
298	}
299
300	/*
301	 * Initialize each kset with a spinlock and delayed work for flushing.
302	 * Each kset is associated with one queue to ensure independent handling
303	 * of cache keys across multiple queues, maximizing multiqueue concurrency.
304	 */
305	for (i = 0; i < cache->n_ksets; i++) {
306		struct pcache_cache_kset *kset = get_kset(cache, i);
307
308		kset->cache = cache;
309		spin_lock_init(&kset->kset_lock);
310		INIT_DELAYED_WORK(&kset->flush_work, kset_flush_fn);
311	}
312
313	cache->data_heads = alloc_percpu(struct pcache_cache_data_head);
314	if (!cache->data_heads) {
315		ret = -ENOMEM;
316		goto free_kset;
317	}
318
319	for_each_possible_cpu(cpu) {
320		struct pcache_cache_data_head *h =
321			per_cpu_ptr(cache->data_heads, cpu);
322		h->head_pos.cache_seg = NULL;
323	}
324
325	/*
326	 * Replay persisted cache keys using cache_replay.
327	 * This function loads and replays cache keys from previously stored
328	 * ksets, allowing the cache to restore its state after a restart.
329	 */
330	ret = cache_replay(cache);
331	if (ret) {
332		pcache_dev_err(pcache, "failed to replay keys\n");
333		goto free_heads;
334	}
335
336	return 0;
337
338free_heads:
339	free_percpu(cache->data_heads);
340free_kset:
341	kvfree(cache->ksets);
342req_tree_exit:
343	cache_tree_exit(&cache->req_key_tree);
344err:
345	return ret;
346}
347
348static void cache_destroy_req_keys(struct pcache_cache *cache)
349{
350	u32 i;
351
352	for (i = 0; i < cache->n_ksets; i++) {
353		struct pcache_cache_kset *kset = get_kset(cache, i);
354
355		cancel_delayed_work_sync(&kset->flush_work);
356	}
357
358	free_percpu(cache->data_heads);
359	kvfree(cache->ksets);
360	cache_tree_exit(&cache->req_key_tree);
361}
362
363int pcache_cache_start(struct dm_pcache *pcache)
364{
365	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
366	struct pcache_cache *cache = &pcache->cache;
367	struct pcache_cache_options *opts = &pcache->opts;
368	int ret;
369
370	ret = cache_init(pcache);
371	if (ret)
372		return ret;
373
374	cache->cache_info_addr = CACHE_DEV_CACHE_INFO(cache->cache_dev);
375	cache->cache_ctrl = CACHE_DEV_CACHE_CTRL(cache->cache_dev);
376	backing_dev->cache = cache;
377	cache->dev_size = backing_dev->dev_size;
378
379	ret = cache_info_init(cache, opts);
380	if (ret)
381		goto cache_exit;
382
383	ret = cache_segs_init(cache);
384	if (ret)
385		goto cache_exit;
386
387	ret = cache_tail_init(cache);
388	if (ret)
389		goto cache_exit;
390
391	ret = cache_init_req_keys(cache, num_online_cpus());
392	if (ret)
393		goto cache_exit;
394
395	ret = cache_writeback_init(cache);
396	if (ret)
397		goto destroy_keys;
398
399	cache->cache_info.flags |= PCACHE_CACHE_FLAGS_INIT_DONE;
400	cache_info_write(cache);
401	queue_delayed_work(cache_get_wq(cache), &cache->gc_work, 0);
402
403	return 0;
404
405destroy_keys:
406	cache_destroy_req_keys(cache);
407cache_exit:
408	cache_exit(cache);
409
410	return ret;
411}
412
413void pcache_cache_stop(struct dm_pcache *pcache)
414{
415	struct pcache_cache *cache = &pcache->cache;
416
417	pcache_cache_flush(cache);
418
419	cancel_delayed_work_sync(&cache->gc_work);
420	flush_work(&cache->clean_work);
421	cache_writeback_exit(cache);
422
423	if (cache->req_key_tree.n_subtrees)
424		cache_destroy_req_keys(cache);
425
426	cache_exit(cache);
427}
428
429struct workqueue_struct *cache_get_wq(struct pcache_cache *cache)
430{
431	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
432
433	return pcache->task_wq;
434}
435
436int pcache_cache_init(void)
437{
438	key_cache = KMEM_CACHE(pcache_cache_key, 0);
439	if (!key_cache)
440		return -ENOMEM;
441
442	return 0;
443}
444
445void pcache_cache_exit(void)
446{
447	kmem_cache_destroy(key_cache);
448}