bio-integrity-auto.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
  4 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
  5 *
  6 * Automatically generate and verify integrity data on PI capable devices if the
  7 * bio submitter didn't provide PI itself.  This ensures that kernel verifies
  8 * data integrity even if the file system (or other user of the block device) is
  9 * not aware of PI.
 10 */
 11#include <linux/blk-integrity.h>
 12#include <linux/t10-pi.h>
 13#include <linux/workqueue.h>
 14#include "blk.h"
 15
 16struct bio_integrity_data {
 17	struct bio			*bio;
 18	struct bvec_iter		saved_bio_iter;
 19	struct work_struct		work;
 20	struct bio_integrity_payload	bip;
 21	struct bio_vec			bvec;
 22};
 23
 24static struct kmem_cache *bid_slab;
 25static mempool_t bid_pool;
 26static struct workqueue_struct *kintegrityd_wq;
 27
 28static void bio_integrity_finish(struct bio_integrity_data *bid)
 29{
 30	bid->bio->bi_integrity = NULL;
 31	bid->bio->bi_opf &= ~REQ_INTEGRITY;
 32	bio_integrity_free_buf(&bid->bip);
 33	mempool_free(bid, &bid_pool);
 34}
 35
 36static void bio_integrity_verify_fn(struct work_struct *work)
 37{
 38	struct bio_integrity_data *bid =
 39		container_of(work, struct bio_integrity_data, work);
 40	struct bio *bio = bid->bio;
 41
 42	blk_integrity_verify_iter(bio, &bid->saved_bio_iter);
 43	bio_integrity_finish(bid);
 44	bio_endio(bio);
 45}
 46
 47#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
 48static bool bip_should_check(struct bio_integrity_payload *bip)
 49{
 50	return bip->bip_flags & BIP_CHECK_FLAGS;
 51}
 52
 53static bool bi_offload_capable(struct blk_integrity *bi)
 54{
 55	switch (bi->csum_type) {
 56	case BLK_INTEGRITY_CSUM_CRC64:
 57		return bi->metadata_size == sizeof(struct crc64_pi_tuple);
 58	case BLK_INTEGRITY_CSUM_CRC:
 59	case BLK_INTEGRITY_CSUM_IP:
 60		return bi->metadata_size == sizeof(struct t10_pi_tuple);
 61	default:
 62		pr_warn_once("%s: unknown integrity checksum type:%d\n",
 63			__func__, bi->csum_type);
 64		fallthrough;
 65	case BLK_INTEGRITY_CSUM_NONE:
 66		return false;
 67	}
 68}
 69
 70/**
 71 * __bio_integrity_endio - Integrity I/O completion function
 72 * @bio:	Protected bio
 73 *
 74 * Normally I/O completion is done in interrupt context.  However, verifying I/O
 75 * integrity is a time-consuming task which must be run in process context.
 76 *
 77 * This function postpones completion accordingly.
 78 */
 79bool __bio_integrity_endio(struct bio *bio)
 80{
 81	struct bio_integrity_payload *bip = bio_integrity(bio);
 82	struct bio_integrity_data *bid =
 83		container_of(bip, struct bio_integrity_data, bip);
 84
 85	if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
 86	    bip_should_check(bip)) {
 87		INIT_WORK(&bid->work, bio_integrity_verify_fn);
 88		queue_work(kintegrityd_wq, &bid->work);
 89		return false;
 90	}
 91
 92	bio_integrity_finish(bid);
 93	return true;
 94}
 95
 96/**
 97 * bio_integrity_prep - Prepare bio for integrity I/O
 98 * @bio:	bio to prepare
 99 *
100 * Checks if the bio already has an integrity payload attached.  If it does, the
101 * payload has been generated by another kernel subsystem, and we just pass it
102 * through.
103 * Otherwise allocates integrity payload and for writes the integrity metadata
104 * will be generated.  For reads, the completion handler will verify the
105 * metadata.
106 */
107bool bio_integrity_prep(struct bio *bio)
108{
109	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
110	struct bio_integrity_data *bid;
111	bool set_flags = true;
112	gfp_t gfp = GFP_NOIO;
113
114	if (!bi)
115		return true;
116
117	if (!bio_sectors(bio))
118		return true;
119
120	/* Already protected? */
121	if (bio_integrity(bio))
122		return true;
123
124	switch (bio_op(bio)) {
125	case REQ_OP_READ:
126		if (bi->flags & BLK_INTEGRITY_NOVERIFY) {
127			if (bi_offload_capable(bi))
128				return true;
129			set_flags = false;
130		}
131		break;
132	case REQ_OP_WRITE:
133		/*
134		 * Zero the memory allocated to not leak uninitialized kernel
135		 * memory to disk for non-integrity metadata where nothing else
136		 * initializes the memory.
137		 */
138		if (bi->flags & BLK_INTEGRITY_NOGENERATE) {
139			if (bi_offload_capable(bi))
140				return true;
141			set_flags = false;
142			gfp |= __GFP_ZERO;
143		} else if (bi->metadata_size > bi->pi_tuple_size)
144			gfp |= __GFP_ZERO;
145		break;
146	default:
147		return true;
148	}
149
150	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
151		return true;
152
153	bid = mempool_alloc(&bid_pool, GFP_NOIO);
154	bio_integrity_init(bio, &bid->bip, &bid->bvec, 1);
155	bid->bio = bio;
156	bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY;
157	bio_integrity_alloc_buf(bio, gfp & __GFP_ZERO);
158
159	bip_set_seed(&bid->bip, bio->bi_iter.bi_sector);
160
161	if (set_flags) {
162		if (bi->csum_type == BLK_INTEGRITY_CSUM_IP)
163			bid->bip.bip_flags |= BIP_IP_CHECKSUM;
164		if (bi->csum_type)
165			bid->bip.bip_flags |= BIP_CHECK_GUARD;
166		if (bi->flags & BLK_INTEGRITY_REF_TAG)
167			bid->bip.bip_flags |= BIP_CHECK_REFTAG;
168	}
169
170	/* Auto-generate integrity metadata if this is a write */
171	if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip))
172		blk_integrity_generate(bio);
173	else
174		bid->saved_bio_iter = bio->bi_iter;
175	return true;
176}
177EXPORT_SYMBOL(bio_integrity_prep);
178
179void blk_flush_integrity(void)
180{
181	flush_workqueue(kintegrityd_wq);
182}
183
184static int __init blk_integrity_auto_init(void)
185{
186	bid_slab = kmem_cache_create("bio_integrity_data",
187			sizeof(struct bio_integrity_data), 0,
188			SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL);
189
190	if (mempool_init_slab_pool(&bid_pool, BIO_POOL_SIZE, bid_slab))
191		panic("bio: can't create integrity pool\n");
192
193	/*
194	 * kintegrityd won't block much but may burn a lot of CPU cycles.
195	 * Make it highpri CPU intensive wq with max concurrency of 1.
196	 */
197	kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
198					 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
199	if (!kintegrityd_wq)
200		panic("Failed to create kintegrityd\n");
201	return 0;
202}
203subsys_initcall(blk_integrity_auto_init);