cache_dev.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#include <linux/blkdev.h>
  4#include <linux/dax.h>
  5#include <linux/vmalloc.h>
  6#include <linux/parser.h>
  7
  8#include "cache_dev.h"
  9#include "backing_dev.h"
 10#include "cache.h"
 11#include "dm_pcache.h"
 12
 13static void cache_dev_dax_exit(struct pcache_cache_dev *cache_dev)
 14{
 15	if (cache_dev->use_vmap)
 16		vunmap(cache_dev->mapping);
 17}
 18
 19static int build_vmap(struct dax_device *dax_dev, long total_pages, void **vaddr)
 20{
 21	struct page **pages;
 22	long i = 0, chunk;
 23	unsigned long pfn;
 24	int ret;
 25
 26	pages = vmalloc_array(total_pages, sizeof(struct page *));
 27	if (!pages)
 28		return -ENOMEM;
 29
 30	do {
 31		chunk = dax_direct_access(dax_dev, i, total_pages - i,
 32					  DAX_ACCESS, NULL, &pfn);
 33		if (chunk <= 0) {
 34			ret = chunk ? chunk : -EINVAL;
 35			goto out_free;
 36		}
 37
 38		if (!pfn_valid(pfn)) {
 39			ret = -EOPNOTSUPP;
 40			goto out_free;
 41		}
 42
 43		while (chunk-- && i < total_pages) {
 44			pages[i++] = pfn_to_page(pfn);
 45			pfn++;
 46			if (!(i & 15))
 47				cond_resched();
 48		}
 49	} while (i < total_pages);
 50
 51	*vaddr = vmap(pages, total_pages, VM_MAP, PAGE_KERNEL);
 52	if (!*vaddr) {
 53		ret = -ENOMEM;
 54		goto out_free;
 55	}
 56
 57	ret = 0;
 58
 59out_free:
 60	vfree(pages);
 61	return ret;
 62}
 63
 64static int cache_dev_dax_init(struct pcache_cache_dev *cache_dev)
 65{
 66	struct dm_pcache	*pcache = CACHE_DEV_TO_PCACHE(cache_dev);
 67	struct dax_device	*dax_dev;
 68	long			total_pages, mapped_pages;
 69	u64			bdev_size;
 70	void			*vaddr;
 71	int			ret;
 72	int			id;
 73	unsigned long		pfn;
 74
 75	dax_dev	= cache_dev->dm_dev->dax_dev;
 76	/* total size check */
 77	bdev_size = bdev_nr_bytes(cache_dev->dm_dev->bdev);
 78	if (bdev_size < PCACHE_CACHE_DEV_SIZE_MIN) {
 79		pcache_dev_err(pcache, "dax device is too small, required at least %llu",
 80				PCACHE_CACHE_DEV_SIZE_MIN);
 81		ret = -ENOSPC;
 82		goto out;
 83	}
 84
 85	total_pages = bdev_size >> PAGE_SHIFT;
 86	/* attempt: direct-map the whole range */
 87	id = dax_read_lock();
 88	mapped_pages = dax_direct_access(dax_dev, 0, total_pages,
 89					 DAX_ACCESS, &vaddr, &pfn);
 90	if (mapped_pages < 0) {
 91		pcache_dev_err(pcache, "dax_direct_access failed: %ld\n", mapped_pages);
 92		ret = mapped_pages;
 93		goto unlock;
 94	}
 95
 96	if (!pfn_valid(pfn)) {
 97		ret = -EOPNOTSUPP;
 98		goto unlock;
 99	}
100
101	if (mapped_pages == total_pages) {
102		/* success: contiguous direct mapping */
103		cache_dev->mapping = vaddr;
104	} else {
105		/* need vmap fallback */
106		ret = build_vmap(dax_dev, total_pages, &vaddr);
107		if (ret) {
108			pcache_dev_err(pcache, "vmap fallback failed: %d\n", ret);
109			goto unlock;
110		}
111
112		cache_dev->mapping	= vaddr;
113		cache_dev->use_vmap	= true;
114	}
115	dax_read_unlock(id);
116
117	return 0;
118unlock:
119	dax_read_unlock(id);
120out:
121	return ret;
122}
123
124void cache_dev_zero_range(struct pcache_cache_dev *cache_dev, void *pos, u32 size)
125{
126	memset(pos, 0, size);
127	dax_flush(cache_dev->dm_dev->dax_dev, pos, size);
128}
129
130static int sb_read(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
131{
132	struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
133
134	if (copy_mc_to_kernel(sb, sb_addr, sizeof(struct pcache_sb)))
135		return -EIO;
136
137	return 0;
138}
139
140static void sb_write(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
141{
142	struct pcache_sb *sb_addr = CACHE_DEV_SB(cache_dev);
143
144	memcpy_flushcache(sb_addr, sb, sizeof(struct pcache_sb));
145	pmem_wmb();
146}
147
148static int sb_init(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
149{
150	struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
151	u64 nr_segs;
152	u64 cache_dev_size;
153	u64 magic;
154	u32 flags = 0;
155
156	magic = le64_to_cpu(sb->magic);
157	if (magic)
158		return -EEXIST;
159
160	cache_dev_size = bdev_nr_bytes(file_bdev(cache_dev->dm_dev->bdev_file));
161	if (cache_dev_size < PCACHE_CACHE_DEV_SIZE_MIN) {
162		pcache_dev_err(pcache, "dax device is too small, required at least %llu",
163				PCACHE_CACHE_DEV_SIZE_MIN);
164		return -ENOSPC;
165	}
166
167	nr_segs = (cache_dev_size - PCACHE_SEGMENTS_OFF) / ((PCACHE_SEG_SIZE));
168
169#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN)
170	flags |= PCACHE_SB_F_BIGENDIAN;
171#endif
172	sb->flags = cpu_to_le32(flags);
173	sb->magic = cpu_to_le64(PCACHE_MAGIC);
174	sb->seg_num = cpu_to_le32(nr_segs);
175	sb->crc = cpu_to_le32(crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4));
176
177	cache_dev_zero_range(cache_dev, CACHE_DEV_CACHE_INFO(cache_dev),
178			     PCACHE_CACHE_INFO_SIZE * PCACHE_META_INDEX_MAX +
179			     PCACHE_CACHE_CTRL_SIZE);
180
181	return 0;
182}
183
184static int sb_validate(struct pcache_cache_dev *cache_dev, struct pcache_sb *sb)
185{
186	struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
187	u32 flags;
188	u32 crc;
189
190	if (le64_to_cpu(sb->magic) != PCACHE_MAGIC) {
191		pcache_dev_err(pcache, "unexpected magic: %llx\n",
192				le64_to_cpu(sb->magic));
193		return -EINVAL;
194	}
195
196	crc = crc32c(PCACHE_CRC_SEED, (void *)(sb) + 4, sizeof(struct pcache_sb) - 4);
197	if (crc != le32_to_cpu(sb->crc)) {
198		pcache_dev_err(pcache, "corrupted sb: %u, expected: %u\n", crc, le32_to_cpu(sb->crc));
199		return -EINVAL;
200	}
201
202	flags = le32_to_cpu(sb->flags);
203#if defined(__BYTE_ORDER) ? (__BIG_ENDIAN == __BYTE_ORDER) : defined(__BIG_ENDIAN)
204	if (!(flags & PCACHE_SB_F_BIGENDIAN)) {
205		pcache_dev_err(pcache, "cache_dev is not big endian\n");
206		return -EINVAL;
207	}
208#else
209	if (flags & PCACHE_SB_F_BIGENDIAN) {
210		pcache_dev_err(pcache, "cache_dev is big endian\n");
211		return -EINVAL;
212	}
213#endif
214	return 0;
215}
216
217static int cache_dev_init(struct pcache_cache_dev *cache_dev, u32 seg_num)
218{
219	cache_dev->seg_num = seg_num;
220	cache_dev->seg_bitmap = kvcalloc(BITS_TO_LONGS(cache_dev->seg_num), sizeof(unsigned long), GFP_KERNEL);
221	if (!cache_dev->seg_bitmap)
222		return -ENOMEM;
223
224	return 0;
225}
226
227static void cache_dev_exit(struct pcache_cache_dev *cache_dev)
228{
229	kvfree(cache_dev->seg_bitmap);
230}
231
232void cache_dev_stop(struct dm_pcache *pcache)
233{
234	struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
235
236	cache_dev_exit(cache_dev);
237	cache_dev_dax_exit(cache_dev);
238}
239
240int cache_dev_start(struct dm_pcache *pcache)
241{
242	struct pcache_cache_dev *cache_dev = &pcache->cache_dev;
243	struct pcache_sb sb;
244	bool format = false;
245	int ret;
246
247	mutex_init(&cache_dev->seg_lock);
248
249	ret = cache_dev_dax_init(cache_dev);
250	if (ret) {
251		pcache_dev_err(pcache, "failed to init cache_dev %s via dax way: %d.",
252			       cache_dev->dm_dev->name, ret);
253		goto err;
254	}
255
256	ret = sb_read(cache_dev, &sb);
257	if (ret)
258		goto dax_release;
259
260	if (le64_to_cpu(sb.magic) == 0) {
261		format = true;
262		ret = sb_init(cache_dev, &sb);
263		if (ret < 0)
264			goto dax_release;
265	}
266
267	ret = sb_validate(cache_dev, &sb);
268	if (ret)
269		goto dax_release;
270
271	cache_dev->sb_flags = le32_to_cpu(sb.flags);
272	ret = cache_dev_init(cache_dev, le32_to_cpu(sb.seg_num));
273	if (ret)
274		goto dax_release;
275
276	if (format)
277		sb_write(cache_dev, &sb);
278
279	return 0;
280
281dax_release:
282	cache_dev_dax_exit(cache_dev);
283err:
284	return ret;
285}
286
287int cache_dev_get_empty_segment_id(struct pcache_cache_dev *cache_dev, u32 *seg_id)
288{
289	int ret;
290
291	mutex_lock(&cache_dev->seg_lock);
292	*seg_id = find_next_zero_bit(cache_dev->seg_bitmap, cache_dev->seg_num, 0);
293	if (*seg_id == cache_dev->seg_num) {
294		ret = -ENOSPC;
295		goto unlock;
296	}
297
298	__set_bit(*seg_id, cache_dev->seg_bitmap);
299	ret = 0;
300unlock:
301	mutex_unlock(&cache_dev->seg_lock);
302	return ret;
303}