bio.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Utility functions for file contents encryption/decryption on
  4 * block device-based filesystems.
  5 *
  6 * Copyright (C) 2015, Google, Inc.
  7 * Copyright (C) 2015, Motorola Mobility
  8 */
  9
 10#include <linux/bio.h>
 11#include <linux/export.h>
 12#include <linux/module.h>
 13#include <linux/namei.h>
 14#include <linux/pagemap.h>
 15
 16#include "fscrypt_private.h"
 17
 18/**
 19 * fscrypt_decrypt_bio() - decrypt the contents of a bio
 20 * @bio: the bio to decrypt
 21 *
 22 * Decrypt the contents of a "read" bio following successful completion of the
 23 * underlying disk read.  The bio must be reading a whole number of blocks of an
 24 * encrypted file directly into the page cache.  If the bio is reading the
 25 * ciphertext into bounce pages instead of the page cache (for example, because
 26 * the file is also compressed, so decompression is required after decryption),
 27 * then this function isn't applicable.  This function may sleep, so it must be
 28 * called from a workqueue rather than from the bio's bi_end_io callback.
 29 *
 30 * Return: %true on success; %false on failure.  On failure, bio->bi_status is
 31 *	   also set to an error status.
 32 */
 33bool fscrypt_decrypt_bio(struct bio *bio)
 34{
 35	struct folio_iter fi;
 36
 37	bio_for_each_folio_all(fi, bio) {
 38		int err = fscrypt_decrypt_pagecache_blocks(fi.folio, fi.length,
 39							   fi.offset);
 40
 41		if (err) {
 42			bio->bi_status = errno_to_blk_status(err);
 43			return false;
 44		}
 45	}
 46	return true;
 47}
 48EXPORT_SYMBOL(fscrypt_decrypt_bio);
 49
 50static int fscrypt_zeroout_range_inline_crypt(const struct inode *inode,
 51					      pgoff_t lblk, sector_t pblk,
 52					      unsigned int len)
 53{
 54	const unsigned int blockbits = inode->i_blkbits;
 55	const unsigned int blocks_per_page = 1 << (PAGE_SHIFT - blockbits);
 56	struct bio *bio;
 57	int ret, err = 0;
 58	int num_pages = 0;
 59
 60	/* This always succeeds since __GFP_DIRECT_RECLAIM is set. */
 61	bio = bio_alloc(inode->i_sb->s_bdev, BIO_MAX_VECS, REQ_OP_WRITE,
 62			GFP_NOFS);
 63
 64	while (len) {
 65		unsigned int blocks_this_page = min(len, blocks_per_page);
 66		unsigned int bytes_this_page = blocks_this_page << blockbits;
 67
 68		if (num_pages == 0) {
 69			fscrypt_set_bio_crypt_ctx(bio, inode, lblk, GFP_NOFS);
 70			bio->bi_iter.bi_sector =
 71					pblk << (blockbits - SECTOR_SHIFT);
 72		}
 73		ret = bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0);
 74		if (WARN_ON_ONCE(ret != bytes_this_page)) {
 75			err = -EIO;
 76			goto out;
 77		}
 78		num_pages++;
 79		len -= blocks_this_page;
 80		lblk += blocks_this_page;
 81		pblk += blocks_this_page;
 82		if (num_pages == BIO_MAX_VECS || !len ||
 83		    !fscrypt_mergeable_bio(bio, inode, lblk)) {
 84			err = submit_bio_wait(bio);
 85			if (err)
 86				goto out;
 87			bio_reset(bio, inode->i_sb->s_bdev, REQ_OP_WRITE);
 88			num_pages = 0;
 89		}
 90	}
 91out:
 92	bio_put(bio);
 93	return err;
 94}
 95
 96/**
 97 * fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file
 98 * @inode: the file's inode
 99 * @lblk: the first file logical block to zero out
100 * @pblk: the first filesystem physical block to zero out
101 * @len: number of blocks to zero out
102 *
103 * Zero out filesystem blocks in an encrypted regular file on-disk, i.e. write
104 * ciphertext blocks which decrypt to the all-zeroes block.  The blocks must be
105 * both logically and physically contiguous.  It's also assumed that the
106 * filesystem only uses a single block device, ->s_bdev.
107 *
108 * Note that since each block uses a different IV, this involves writing a
109 * different ciphertext to each block; we can't simply reuse the same one.
110 *
111 * Return: 0 on success; -errno on failure.
112 */
113int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
114			  sector_t pblk, unsigned int len)
115{
116	const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode);
117	const unsigned int du_bits = ci->ci_data_unit_bits;
118	const unsigned int du_size = 1U << du_bits;
119	const unsigned int du_per_page_bits = PAGE_SHIFT - du_bits;
120	const unsigned int du_per_page = 1U << du_per_page_bits;
121	u64 du_index = (u64)lblk << (inode->i_blkbits - du_bits);
122	u64 du_remaining = (u64)len << (inode->i_blkbits - du_bits);
123	sector_t sector = pblk << (inode->i_blkbits - SECTOR_SHIFT);
124	struct page *pages[16]; /* write up to 16 pages at a time */
125	unsigned int nr_pages;
126	unsigned int i;
127	unsigned int offset;
128	struct bio *bio;
129	int ret, err;
130
131	if (len == 0)
132		return 0;
133
134	if (fscrypt_inode_uses_inline_crypto(inode))
135		return fscrypt_zeroout_range_inline_crypt(inode, lblk, pblk,
136							  len);
137
138	BUILD_BUG_ON(ARRAY_SIZE(pages) > BIO_MAX_VECS);
139	nr_pages = min_t(u64, ARRAY_SIZE(pages),
140			 (du_remaining + du_per_page - 1) >> du_per_page_bits);
141
142	/*
143	 * We need at least one page for ciphertext.  Allocate the first one
144	 * from a mempool, with __GFP_DIRECT_RECLAIM set so that it can't fail.
145	 *
146	 * Any additional page allocations are allowed to fail, as they only
147	 * help performance, and waiting on the mempool for them could deadlock.
148	 */
149	for (i = 0; i < nr_pages; i++) {
150		pages[i] = fscrypt_alloc_bounce_page(i == 0 ? GFP_NOFS :
151						     GFP_NOWAIT);
152		if (!pages[i])
153			break;
154	}
155	nr_pages = i;
156	if (WARN_ON_ONCE(nr_pages <= 0))
157		return -EINVAL;
158
159	/* This always succeeds since __GFP_DIRECT_RECLAIM is set. */
160	bio = bio_alloc(inode->i_sb->s_bdev, nr_pages, REQ_OP_WRITE, GFP_NOFS);
161
162	do {
163		bio->bi_iter.bi_sector = sector;
164
165		i = 0;
166		offset = 0;
167		do {
168			err = fscrypt_crypt_data_unit(ci, FS_ENCRYPT, du_index,
169						      ZERO_PAGE(0), pages[i],
170						      du_size, offset);
171			if (err)
172				goto out;
173			du_index++;
174			sector += 1U << (du_bits - SECTOR_SHIFT);
175			du_remaining--;
176			offset += du_size;
177			if (offset == PAGE_SIZE || du_remaining == 0) {
178				ret = bio_add_page(bio, pages[i++], offset, 0);
179				if (WARN_ON_ONCE(ret != offset)) {
180					err = -EIO;
181					goto out;
182				}
183				offset = 0;
184			}
185		} while (i != nr_pages && du_remaining != 0);
186
187		err = submit_bio_wait(bio);
188		if (err)
189			goto out;
190		bio_reset(bio, inode->i_sb->s_bdev, REQ_OP_WRITE);
191	} while (du_remaining != 0);
192	err = 0;
193out:
194	bio_put(bio);
195	for (i = 0; i < nr_pages; i++)
196		fscrypt_free_bounce_page(pages[i]);
197	return err;
198}
199EXPORT_SYMBOL(fscrypt_zeroout_range);