dm-log-writes.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2014 Facebook. All rights reserved.
  4 *
  5 * This file is released under the GPL.
  6 */
  7
  8#include <linux/device-mapper.h>
  9
 10#include <linux/module.h>
 11#include <linux/init.h>
 12#include <linux/blkdev.h>
 13#include <linux/bio.h>
 14#include <linux/dax.h>
 15#include <linux/slab.h>
 16#include <linux/kthread.h>
 17#include <linux/freezer.h>
 18#include <linux/uio.h>
 19
 20#define DM_MSG_PREFIX "log-writes"
 21
 22/*
 23 * This target will sequentially log all writes to the target device onto the
 24 * log device.  This is helpful for replaying writes to check for fs consistency
 25 * at all times.  This target provides a mechanism to mark specific events to
 26 * check data at a later time.  So for example you would:
 27 *
 28 * write data
 29 * fsync
 30 * dmsetup message /dev/whatever mark mymark
 31 * unmount /mnt/test
 32 *
 33 * Then replay the log up to mymark and check the contents of the replay to
 34 * verify it matches what was written.
 35 *
 36 * We log writes only after they have been flushed, this makes the log describe
 37 * close to the order in which the data hits the actual disk, not its cache.  So
 38 * for example the following sequence (W means write, C means complete)
 39 *
 40 * Wa,Wb,Wc,Cc,Ca,FLUSH,FUAd,Cb,CFLUSH,CFUAd
 41 *
 42 * Would result in the log looking like this:
 43 *
 44 * c,a,b,flush,fuad,<other writes>,<next flush>
 45 *
 46 * This is meant to help expose problems where file systems do not properly wait
 47 * on data being written before invoking a FLUSH.  FUA bypasses cache so once it
 48 * completes it is added to the log as it should be on disk.
 49 *
 50 * We treat DISCARDs as if they don't bypass cache so that they are logged in
 51 * order of completion along with the normal writes.  If we didn't do it this
 52 * way we would process all the discards first and then write all the data, when
 53 * in fact we want to do the data and the discard in the order that they
 54 * completed.
 55 */
 56#define LOG_FLUSH_FLAG		(1 << 0)
 57#define LOG_FUA_FLAG		(1 << 1)
 58#define LOG_DISCARD_FLAG	(1 << 2)
 59#define LOG_MARK_FLAG		(1 << 3)
 60#define LOG_METADATA_FLAG	(1 << 4)
 61
 62#define WRITE_LOG_VERSION 1ULL
 63#define WRITE_LOG_MAGIC 0x6a736677736872ULL
 64#define WRITE_LOG_SUPER_SECTOR 0
 65
 66/*
 67 * The disk format for this is braindead simple.
 68 *
 69 * At byte 0 we have our super, followed by the following sequence for
 70 * nr_entries:
 71 *
 72 * [   1 sector    ][  entry->nr_sectors ]
 73 * [log_write_entry][    data written    ]
 74 *
 75 * The log_write_entry takes up a full sector so we can have arbitrary length
 76 * marks and it leaves us room for extra content in the future.
 77 */
 78
 79/*
 80 * Basic info about the log for userspace.
 81 */
 82struct log_write_super {
 83	__le64 magic;
 84	__le64 version;
 85	__le64 nr_entries;
 86	__le32 sectorsize;
 87};
 88
 89/*
 90 * sector - the sector we wrote.
 91 * nr_sectors - the number of sectors we wrote.
 92 * flags - flags for this log entry.
 93 * data_len - the size of the data in this log entry, this is for private log
 94 * entry stuff, the MARK data provided by userspace for example.
 95 */
 96struct log_write_entry {
 97	__le64 sector;
 98	__le64 nr_sectors;
 99	__le64 flags;
100	__le64 data_len;
101};
102
103struct log_writes_c {
104	struct dm_dev *dev;
105	struct dm_dev *logdev;
106	u64 logged_entries;
107	u32 sectorsize;
108	u32 sectorshift;
109	atomic_t io_blocks;
110	atomic_t pending_blocks;
111	sector_t next_sector;
112	sector_t end_sector;
113	bool logging_enabled;
114	bool device_supports_discard;
115	spinlock_t blocks_lock;
116	struct list_head unflushed_blocks;
117	struct list_head logging_blocks;
118	wait_queue_head_t wait;
119	struct task_struct *log_kthread;
120	struct completion super_done;
121};
122
123struct pending_block {
124	int vec_cnt;
125	u64 flags;
126	sector_t sector;
127	sector_t nr_sectors;
128	char *data;
129	u32 datalen;
130	struct list_head list;
131	struct bio_vec vecs[];
132};
133
134struct per_bio_data {
135	struct pending_block *block;
136};
137
138static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,
139					  sector_t sectors)
140{
141	return sectors >> (lc->sectorshift - SECTOR_SHIFT);
142}
143
144static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,
145					  sector_t sectors)
146{
147	return sectors << (lc->sectorshift - SECTOR_SHIFT);
148}
149
150static void put_pending_block(struct log_writes_c *lc)
151{
152	if (atomic_dec_and_test(&lc->pending_blocks)) {
153		smp_mb__after_atomic();
154		if (waitqueue_active(&lc->wait))
155			wake_up(&lc->wait);
156	}
157}
158
159static void put_io_block(struct log_writes_c *lc)
160{
161	if (atomic_dec_and_test(&lc->io_blocks)) {
162		smp_mb__after_atomic();
163		if (waitqueue_active(&lc->wait))
164			wake_up(&lc->wait);
165	}
166}
167
168static void log_end_io(struct bio *bio)
169{
170	struct log_writes_c *lc = bio->bi_private;
171
172	if (bio->bi_status) {
173		unsigned long flags;
174
175		DMERR("Error writing log block, error=%d", bio->bi_status);
176		spin_lock_irqsave(&lc->blocks_lock, flags);
177		lc->logging_enabled = false;
178		spin_unlock_irqrestore(&lc->blocks_lock, flags);
179	}
180
181	bio_free_pages(bio);
182	put_io_block(lc);
183	bio_put(bio);
184}
185
186static void log_end_super(struct bio *bio)
187{
188	struct log_writes_c *lc = bio->bi_private;
189
190	complete(&lc->super_done);
191	log_end_io(bio);
192}
193
194/*
195 * Meant to be called if there is an error, it will free all the pages
196 * associated with the block.
197 */
198static void free_pending_block(struct log_writes_c *lc,
199			       struct pending_block *block)
200{
201	int i;
202
203	for (i = 0; i < block->vec_cnt; i++) {
204		if (block->vecs[i].bv_page)
205			__free_page(block->vecs[i].bv_page);
206	}
207	kfree(block->data);
208	kfree(block);
209	put_pending_block(lc);
210}
211
212static int write_metadata(struct log_writes_c *lc, void *entry,
213			  size_t entrylen, void *data, size_t datalen,
214			  sector_t sector)
215{
216	struct bio *bio;
217	struct page *page;
218	void *ptr;
219	size_t ret;
220
221	bio = bio_alloc(lc->logdev->bdev, 1, REQ_OP_WRITE, GFP_KERNEL);
222	bio->bi_iter.bi_size = 0;
223	bio->bi_iter.bi_sector = sector;
224	bio->bi_end_io = (sector == WRITE_LOG_SUPER_SECTOR) ?
225			  log_end_super : log_end_io;
226	bio->bi_private = lc;
227
228	page = alloc_page(GFP_KERNEL);
229	if (!page) {
230		DMERR("Couldn't alloc log page");
231		bio_put(bio);
232		goto error;
233	}
234
235	ptr = kmap_local_page(page);
236	memcpy(ptr, entry, entrylen);
237	if (datalen)
238		memcpy(ptr + entrylen, data, datalen);
239	memset(ptr + entrylen + datalen, 0,
240	       lc->sectorsize - entrylen - datalen);
241	kunmap_local(ptr);
242
243	ret = bio_add_page(bio, page, lc->sectorsize, 0);
244	if (ret != lc->sectorsize) {
245		DMERR("Couldn't add page to the log block");
246		goto error_bio;
247	}
248	submit_bio(bio);
249	return 0;
250error_bio:
251	bio_put(bio);
252	__free_page(page);
253error:
254	put_io_block(lc);
255	return -1;
256}
257
258static int write_inline_data(struct log_writes_c *lc, void *entry,
259			     size_t entrylen, void *data, size_t datalen,
260			     sector_t sector)
261{
262	int bio_pages, pg_datalen, pg_sectorlen, i;
263	struct page *page;
264	struct bio *bio;
265	size_t ret;
266	void *ptr;
267
268	while (datalen) {
269		bio_pages = bio_max_segs(DIV_ROUND_UP(datalen, PAGE_SIZE));
270
271		atomic_inc(&lc->io_blocks);
272
273		bio = bio_alloc(lc->logdev->bdev, bio_pages, REQ_OP_WRITE,
274				GFP_KERNEL);
275		bio->bi_iter.bi_size = 0;
276		bio->bi_iter.bi_sector = sector;
277		bio->bi_end_io = log_end_io;
278		bio->bi_private = lc;
279
280		for (i = 0; i < bio_pages; i++) {
281			pg_datalen = min_t(int, datalen, PAGE_SIZE);
282			pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);
283
284			page = alloc_page(GFP_KERNEL);
285			if (!page) {
286				DMERR("Couldn't alloc inline data page");
287				goto error_bio;
288			}
289
290			ptr = kmap_local_page(page);
291			memcpy(ptr, data, pg_datalen);
292			if (pg_sectorlen > pg_datalen)
293				memset(ptr + pg_datalen, 0, pg_sectorlen - pg_datalen);
294			kunmap_local(ptr);
295
296			ret = bio_add_page(bio, page, pg_sectorlen, 0);
297			if (ret != pg_sectorlen) {
298				DMERR("Couldn't add page of inline data");
299				__free_page(page);
300				goto error_bio;
301			}
302
303			datalen -= pg_datalen;
304			data	+= pg_datalen;
305		}
306		submit_bio(bio);
307
308		sector += bio_pages * PAGE_SECTORS;
309	}
310	return 0;
311error_bio:
312	bio_free_pages(bio);
313	bio_put(bio);
314	put_io_block(lc);
315	return -1;
316}
317
318static int log_one_block(struct log_writes_c *lc,
319			 struct pending_block *block, sector_t sector)
320{
321	struct bio *bio;
322	struct log_write_entry entry;
323	size_t metadatalen, ret;
324	int i;
325
326	entry.sector = cpu_to_le64(block->sector);
327	entry.nr_sectors = cpu_to_le64(block->nr_sectors);
328	entry.flags = cpu_to_le64(block->flags);
329	entry.data_len = cpu_to_le64(block->datalen);
330
331	metadatalen = (block->flags & LOG_MARK_FLAG) ? block->datalen : 0;
332	if (write_metadata(lc, &entry, sizeof(entry), block->data,
333			   metadatalen, sector)) {
334		free_pending_block(lc, block);
335		return -1;
336	}
337
338	sector += dev_to_bio_sectors(lc, 1);
339
340	if (block->datalen && metadatalen == 0) {
341		if (write_inline_data(lc, &entry, sizeof(entry), block->data,
342				      block->datalen, sector)) {
343			free_pending_block(lc, block);
344			return -1;
345		}
346		/* we don't support both inline data & bio data */
347		goto out;
348	}
349
350	if (!block->vec_cnt)
351		goto out;
352
353	atomic_inc(&lc->io_blocks);
354	bio = bio_alloc(lc->logdev->bdev, bio_max_segs(block->vec_cnt),
355			REQ_OP_WRITE, GFP_KERNEL);
356	bio->bi_iter.bi_size = 0;
357	bio->bi_iter.bi_sector = sector;
358	bio->bi_end_io = log_end_io;
359	bio->bi_private = lc;
360
361	for (i = 0; i < block->vec_cnt; i++) {
362		/*
363		 * The page offset is always 0 because we allocate a new page
364		 * for every bvec in the original bio for simplicity sake.
365		 */
366		ret = bio_add_page(bio, block->vecs[i].bv_page,
367				   block->vecs[i].bv_len, 0);
368		if (ret != block->vecs[i].bv_len) {
369			atomic_inc(&lc->io_blocks);
370			submit_bio(bio);
371			bio = bio_alloc(lc->logdev->bdev,
372					bio_max_segs(block->vec_cnt - i),
373					REQ_OP_WRITE, GFP_KERNEL);
374			bio->bi_iter.bi_size = 0;
375			bio->bi_iter.bi_sector = sector;
376			bio->bi_end_io = log_end_io;
377			bio->bi_private = lc;
378
379			ret = bio_add_page(bio, block->vecs[i].bv_page,
380					   block->vecs[i].bv_len, 0);
381			if (ret != block->vecs[i].bv_len) {
382				DMERR("Couldn't add page on new bio?");
383				bio_put(bio);
384				goto error;
385			}
386		}
387		sector += block->vecs[i].bv_len >> SECTOR_SHIFT;
388	}
389	submit_bio(bio);
390out:
391	kfree(block->data);
392	kfree(block);
393	put_pending_block(lc);
394	return 0;
395error:
396	free_pending_block(lc, block);
397	put_io_block(lc);
398	return -1;
399}
400
401static int log_super(struct log_writes_c *lc)
402{
403	struct log_write_super super;
404
405	super.magic = cpu_to_le64(WRITE_LOG_MAGIC);
406	super.version = cpu_to_le64(WRITE_LOG_VERSION);
407	super.nr_entries = cpu_to_le64(lc->logged_entries);
408	super.sectorsize = cpu_to_le32(lc->sectorsize);
409
410	if (write_metadata(lc, &super, sizeof(super), NULL, 0,
411			   WRITE_LOG_SUPER_SECTOR)) {
412		DMERR("Couldn't write super");
413		return -1;
414	}
415
416	/*
417	 * Super sector should be written in-order, otherwise the
418	 * nr_entries could be rewritten incorrectly by an old bio.
419	 */
420	wait_for_completion_io(&lc->super_done);
421
422	return 0;
423}
424
425static inline sector_t logdev_last_sector(struct log_writes_c *lc)
426{
427	return bdev_nr_sectors(lc->logdev->bdev);
428}
429
430static int log_writes_kthread(void *arg)
431{
432	struct log_writes_c *lc = arg;
433	sector_t sector = 0;
434
435	set_freezable();
436	while (!kthread_should_stop()) {
437		bool super = false;
438		bool logging_enabled;
439		struct pending_block *block = NULL;
440		int ret;
441
442		spin_lock_irq(&lc->blocks_lock);
443		if (!list_empty(&lc->logging_blocks)) {
444			block = list_first_entry(&lc->logging_blocks,
445						 struct pending_block, list);
446			list_del_init(&block->list);
447			if (!lc->logging_enabled)
448				goto next;
449
450			sector = lc->next_sector;
451			if (!(block->flags & LOG_DISCARD_FLAG))
452				lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);
453			lc->next_sector += dev_to_bio_sectors(lc, 1);
454
455			/*
456			 * Apparently the size of the device may not be known
457			 * right away, so handle this properly.
458			 */
459			if (!lc->end_sector)
460				lc->end_sector = logdev_last_sector(lc);
461			if (lc->end_sector &&
462			    lc->next_sector >= lc->end_sector) {
463				DMERR("Ran out of space on the logdev");
464				lc->logging_enabled = false;
465				goto next;
466			}
467			lc->logged_entries++;
468			atomic_inc(&lc->io_blocks);
469
470			super = (block->flags & (LOG_FUA_FLAG | LOG_MARK_FLAG));
471			if (super)
472				atomic_inc(&lc->io_blocks);
473		}
474next:
475		logging_enabled = lc->logging_enabled;
476		spin_unlock_irq(&lc->blocks_lock);
477		if (block) {
478			if (logging_enabled) {
479				ret = log_one_block(lc, block, sector);
480				if (!ret && super)
481					ret = log_super(lc);
482				if (ret) {
483					spin_lock_irq(&lc->blocks_lock);
484					lc->logging_enabled = false;
485					spin_unlock_irq(&lc->blocks_lock);
486				}
487			} else
488				free_pending_block(lc, block);
489			continue;
490		}
491
492		if (!try_to_freeze()) {
493			set_current_state(TASK_INTERRUPTIBLE);
494			if (!kthread_should_stop() &&
495			    list_empty(&lc->logging_blocks))
496				schedule();
497			__set_current_state(TASK_RUNNING);
498		}
499	}
500	return 0;
501}
502
503/*
504 * Construct a log-writes mapping:
505 * log-writes <dev_path> <log_dev_path>
506 */
507static int log_writes_ctr(struct dm_target *ti, unsigned int argc, char **argv)
508{
509	struct log_writes_c *lc;
510	struct dm_arg_set as;
511	const char *devname, *logdevname;
512	int ret;
513
514	as.argc = argc;
515	as.argv = argv;
516
517	if (argc < 2) {
518		ti->error = "Invalid argument count";
519		return -EINVAL;
520	}
521
522	lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);
523	if (!lc) {
524		ti->error = "Cannot allocate context";
525		return -ENOMEM;
526	}
527	spin_lock_init(&lc->blocks_lock);
528	INIT_LIST_HEAD(&lc->unflushed_blocks);
529	INIT_LIST_HEAD(&lc->logging_blocks);
530	init_waitqueue_head(&lc->wait);
531	init_completion(&lc->super_done);
532	atomic_set(&lc->io_blocks, 0);
533	atomic_set(&lc->pending_blocks, 0);
534
535	devname = dm_shift_arg(&as);
536	ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);
537	if (ret) {
538		ti->error = "Device lookup failed";
539		goto bad;
540	}
541
542	logdevname = dm_shift_arg(&as);
543	ret = dm_get_device(ti, logdevname, dm_table_get_mode(ti->table),
544			    &lc->logdev);
545	if (ret) {
546		ti->error = "Log device lookup failed";
547		dm_put_device(ti, lc->dev);
548		goto bad;
549	}
550
551	lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);
552	lc->sectorshift = ilog2(lc->sectorsize);
553	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");
554	if (IS_ERR(lc->log_kthread)) {
555		ret = PTR_ERR(lc->log_kthread);
556		ti->error = "Couldn't alloc kthread";
557		dm_put_device(ti, lc->dev);
558		dm_put_device(ti, lc->logdev);
559		goto bad;
560	}
561
562	/*
563	 * next_sector is in 512b sectors to correspond to what bi_sector expects.
564	 * The super starts at sector 0, and the next_sector is the next logical
565	 * one based on the sectorsize of the device.
566	 */
567	lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;
568	lc->logging_enabled = true;
569	lc->end_sector = logdev_last_sector(lc);
570	lc->device_supports_discard = true;
571
572	ti->num_flush_bios = 1;
573	ti->flush_supported = true;
574	ti->num_discard_bios = 1;
575	ti->discards_supported = true;
576	ti->per_io_data_size = sizeof(struct per_bio_data);
577	ti->private = lc;
578	return 0;
579
580bad:
581	kfree(lc);
582	return ret;
583}
584
585static int log_mark(struct log_writes_c *lc, char *data)
586{
587	struct pending_block *block;
588	size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);
589
590	block = kzalloc(sizeof(struct pending_block), GFP_KERNEL);
591	if (!block) {
592		DMERR("Error allocating pending block");
593		return -ENOMEM;
594	}
595
596	block->data = kstrndup(data, maxsize - 1, GFP_KERNEL);
597	if (!block->data) {
598		DMERR("Error copying mark data");
599		kfree(block);
600		return -ENOMEM;
601	}
602	atomic_inc(&lc->pending_blocks);
603	block->datalen = strlen(block->data);
604	block->flags |= LOG_MARK_FLAG;
605	spin_lock_irq(&lc->blocks_lock);
606	list_add_tail(&block->list, &lc->logging_blocks);
607	spin_unlock_irq(&lc->blocks_lock);
608	wake_up_process(lc->log_kthread);
609	return 0;
610}
611
612static void log_writes_dtr(struct dm_target *ti)
613{
614	struct log_writes_c *lc = ti->private;
615
616	spin_lock_irq(&lc->blocks_lock);
617	list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);
618	spin_unlock_irq(&lc->blocks_lock);
619
620	/*
621	 * This is just nice to have since it'll update the super to include the
622	 * unflushed blocks, if it fails we don't really care.
623	 */
624	log_mark(lc, "dm-log-writes-end");
625	wake_up_process(lc->log_kthread);
626	wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&
627		   !atomic_read(&lc->pending_blocks));
628	kthread_stop(lc->log_kthread);
629
630	WARN_ON(!list_empty(&lc->logging_blocks));
631	WARN_ON(!list_empty(&lc->unflushed_blocks));
632	dm_put_device(ti, lc->dev);
633	dm_put_device(ti, lc->logdev);
634	kfree(lc);
635}
636
637static void normal_map_bio(struct dm_target *ti, struct bio *bio)
638{
639	struct log_writes_c *lc = ti->private;
640
641	bio_set_dev(bio, lc->dev->bdev);
642}
643
644static int log_writes_map(struct dm_target *ti, struct bio *bio)
645{
646	struct log_writes_c *lc = ti->private;
647	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
648	struct pending_block *block;
649	struct bvec_iter iter;
650	struct bio_vec bv;
651	size_t alloc_size;
652	int i = 0;
653	bool flush_bio = (bio->bi_opf & REQ_PREFLUSH);
654	bool fua_bio = (bio->bi_opf & REQ_FUA);
655	bool discard_bio = (bio_op(bio) == REQ_OP_DISCARD);
656	bool meta_bio = (bio->bi_opf & REQ_META);
657
658	pb->block = NULL;
659
660	/* Don't bother doing anything if logging has been disabled */
661	if (!lc->logging_enabled)
662		goto map_bio;
663
664	/*
665	 * Map reads as normal.
666	 */
667	if (bio_data_dir(bio) == READ)
668		goto map_bio;
669
670	/* No sectors and not a flush?  Don't care */
671	if (!bio_sectors(bio) && !flush_bio)
672		goto map_bio;
673
674	/*
675	 * Discards will have bi_size set but there's no actual data, so just
676	 * allocate the size of the pending block.
677	 */
678	if (discard_bio)
679		alloc_size = sizeof(struct pending_block);
680	else
681		alloc_size = struct_size(block, vecs, bio_segments(bio));
682
683	block = kzalloc(alloc_size, GFP_NOIO);
684	if (!block) {
685		DMERR("Error allocating pending block");
686		spin_lock_irq(&lc->blocks_lock);
687		lc->logging_enabled = false;
688		spin_unlock_irq(&lc->blocks_lock);
689		return DM_MAPIO_KILL;
690	}
691	INIT_LIST_HEAD(&block->list);
692	pb->block = block;
693	atomic_inc(&lc->pending_blocks);
694
695	if (flush_bio)
696		block->flags |= LOG_FLUSH_FLAG;
697	if (fua_bio)
698		block->flags |= LOG_FUA_FLAG;
699	if (discard_bio)
700		block->flags |= LOG_DISCARD_FLAG;
701	if (meta_bio)
702		block->flags |= LOG_METADATA_FLAG;
703
704	block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);
705	block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));
706
707	/* We don't need the data, just submit */
708	if (discard_bio) {
709		WARN_ON(flush_bio || fua_bio);
710		if (lc->device_supports_discard)
711			goto map_bio;
712		bio_endio(bio);
713		return DM_MAPIO_SUBMITTED;
714	}
715
716	/* Flush bio, splice the unflushed blocks onto this list and submit */
717	if (flush_bio && !bio_sectors(bio)) {
718		spin_lock_irq(&lc->blocks_lock);
719		list_splice_init(&lc->unflushed_blocks, &block->list);
720		spin_unlock_irq(&lc->blocks_lock);
721		goto map_bio;
722	}
723
724	/*
725	 * We will write this bio somewhere else way later so we need to copy
726	 * the actual contents into new pages so we know the data will always be
727	 * there.
728	 *
729	 * We do this because this could be a bio from O_DIRECT in which case we
730	 * can't just hold onto the page until some later point, we have to
731	 * manually copy the contents.
732	 */
733	bio_for_each_segment(bv, bio, iter) {
734		struct page *page;
735		void *dst;
736
737		page = alloc_page(GFP_NOIO);
738		if (!page) {
739			DMERR("Error allocing page");
740			free_pending_block(lc, block);
741			spin_lock_irq(&lc->blocks_lock);
742			lc->logging_enabled = false;
743			spin_unlock_irq(&lc->blocks_lock);
744			return DM_MAPIO_KILL;
745		}
746
747		dst = kmap_local_page(page);
748		memcpy_from_bvec(dst, &bv);
749		kunmap_local(dst);
750		block->vecs[i].bv_page = page;
751		block->vecs[i].bv_len = bv.bv_len;
752		block->vec_cnt++;
753		i++;
754	}
755
756	/* Had a flush with data in it, weird */
757	if (flush_bio) {
758		spin_lock_irq(&lc->blocks_lock);
759		list_splice_init(&lc->unflushed_blocks, &block->list);
760		spin_unlock_irq(&lc->blocks_lock);
761	}
762map_bio:
763	normal_map_bio(ti, bio);
764	return DM_MAPIO_REMAPPED;
765}
766
767static int normal_end_io(struct dm_target *ti, struct bio *bio,
768		blk_status_t *error)
769{
770	struct log_writes_c *lc = ti->private;
771	struct per_bio_data *pb = dm_per_bio_data(bio, sizeof(struct per_bio_data));
772
773	if (bio_data_dir(bio) == WRITE && pb->block) {
774		struct pending_block *block = pb->block;
775		unsigned long flags;
776
777		spin_lock_irqsave(&lc->blocks_lock, flags);
778		if (block->flags & LOG_FLUSH_FLAG) {
779			list_splice_tail_init(&block->list, &lc->logging_blocks);
780			list_add_tail(&block->list, &lc->logging_blocks);
781			wake_up_process(lc->log_kthread);
782		} else if (block->flags & LOG_FUA_FLAG) {
783			list_add_tail(&block->list, &lc->logging_blocks);
784			wake_up_process(lc->log_kthread);
785		} else
786			list_add_tail(&block->list, &lc->unflushed_blocks);
787		spin_unlock_irqrestore(&lc->blocks_lock, flags);
788	}
789
790	return DM_ENDIO_DONE;
791}
792
793/*
794 * INFO format: <logged entries> <highest allocated sector>
795 */
796static void log_writes_status(struct dm_target *ti, status_type_t type,
797			      unsigned int status_flags, char *result,
798			      unsigned int maxlen)
799{
800	unsigned int sz = 0;
801	struct log_writes_c *lc = ti->private;
802
803	switch (type) {
804	case STATUSTYPE_INFO:
805		DMEMIT("%llu %llu", lc->logged_entries,
806		       (unsigned long long)lc->next_sector - 1);
807		if (!lc->logging_enabled)
808			DMEMIT(" logging_disabled");
809		break;
810
811	case STATUSTYPE_TABLE:
812		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);
813		break;
814
815	case STATUSTYPE_IMA:
816		*result = '\0';
817		break;
818	}
819}
820
821static int log_writes_prepare_ioctl(struct dm_target *ti,
822				    struct block_device **bdev,
823				    unsigned int cmd, unsigned long arg,
824				    bool *forward)
825{
826	struct log_writes_c *lc = ti->private;
827	struct dm_dev *dev = lc->dev;
828
829	*bdev = dev->bdev;
830	/*
831	 * Only pass ioctls through if the device sizes match exactly.
832	 */
833	if (ti->len != bdev_nr_sectors(dev->bdev))
834		return 1;
835	return 0;
836}
837
838static int log_writes_iterate_devices(struct dm_target *ti,
839				      iterate_devices_callout_fn fn,
840				      void *data)
841{
842	struct log_writes_c *lc = ti->private;
843
844	return fn(ti, lc->dev, 0, ti->len, data);
845}
846
847/*
848 * Messages supported:
849 *   mark <mark data> - specify the marked data.
850 */
851static int log_writes_message(struct dm_target *ti, unsigned int argc, char **argv,
852			      char *result, unsigned int maxlen)
853{
854	int r = -EINVAL;
855	struct log_writes_c *lc = ti->private;
856
857	if (argc != 2) {
858		DMWARN("Invalid log-writes message arguments, expect 2 arguments, got %d", argc);
859		return r;
860	}
861
862	if (!strcasecmp(argv[0], "mark"))
863		r = log_mark(lc, argv[1]);
864	else
865		DMWARN("Unrecognised log writes target message received: %s", argv[0]);
866
867	return r;
868}
869
870static void log_writes_io_hints(struct dm_target *ti, struct queue_limits *limits)
871{
872	struct log_writes_c *lc = ti->private;
873
874	if (!bdev_max_discard_sectors(lc->dev->bdev)) {
875		lc->device_supports_discard = false;
876		limits->discard_granularity = lc->sectorsize;
877		limits->max_hw_discard_sectors = (UINT_MAX >> SECTOR_SHIFT);
878	}
879	limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);
880	limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);
881	limits->io_min = limits->physical_block_size;
882	limits->dma_alignment = limits->logical_block_size - 1;
883}
884
885#if IS_ENABLED(CONFIG_FS_DAX)
886static struct dax_device *log_writes_dax_pgoff(struct dm_target *ti,
887		pgoff_t *pgoff)
888{
889	struct log_writes_c *lc = ti->private;
890
891	*pgoff += (get_start_sect(lc->dev->bdev) >> PAGE_SECTORS_SHIFT);
892	return lc->dev->dax_dev;
893}
894
895static long log_writes_dax_direct_access(struct dm_target *ti, pgoff_t pgoff,
896		long nr_pages, enum dax_access_mode mode, void **kaddr,
897		unsigned long *pfn)
898{
899	struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
900
901	return dax_direct_access(dax_dev, pgoff, nr_pages, mode, kaddr, pfn);
902}
903
904static int log_writes_dax_zero_page_range(struct dm_target *ti, pgoff_t pgoff,
905					  size_t nr_pages)
906{
907	struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
908
909	return dax_zero_page_range(dax_dev, pgoff, nr_pages << PAGE_SHIFT);
910}
911
912static size_t log_writes_dax_recovery_write(struct dm_target *ti,
913		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
914{
915	struct dax_device *dax_dev = log_writes_dax_pgoff(ti, &pgoff);
916
917	return dax_recovery_write(dax_dev, pgoff, addr, bytes, i);
918}
919
920#else
921#define log_writes_dax_direct_access NULL
922#define log_writes_dax_zero_page_range NULL
923#define log_writes_dax_recovery_write NULL
924#endif
925
926static struct target_type log_writes_target = {
927	.name   = "log-writes",
928	.version = {1, 1, 0},
929	.module = THIS_MODULE,
930	.ctr    = log_writes_ctr,
931	.dtr    = log_writes_dtr,
932	.map    = log_writes_map,
933	.end_io = normal_end_io,
934	.status = log_writes_status,
935	.prepare_ioctl = log_writes_prepare_ioctl,
936	.message = log_writes_message,
937	.iterate_devices = log_writes_iterate_devices,
938	.io_hints = log_writes_io_hints,
939	.direct_access = log_writes_dax_direct_access,
940	.dax_zero_page_range = log_writes_dax_zero_page_range,
941	.dax_recovery_write = log_writes_dax_recovery_write,
942};
943module_dm(log_writes);
944
945MODULE_DESCRIPTION(DM_NAME " log writes target");
946MODULE_AUTHOR("Josef Bacik <jbacik@fb.com>");
947MODULE_LICENSE("GPL");