io-factory.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright 2023 Red Hat
  4 */
  5
  6#include "io-factory.h"
  7
  8#include <linux/atomic.h>
  9#include <linux/blkdev.h>
 10#include <linux/err.h>
 11#include <linux/mount.h>
 12
 13#include "logger.h"
 14#include "memory-alloc.h"
 15#include "numeric.h"
 16
 17/*
 18 * The I/O factory object manages access to index storage, which is a contiguous range of blocks on
 19 * a block device.
 20 *
 21 * The factory holds the open device and is responsible for closing it. The factory has methods to
 22 * make helper structures that can be used to access sections of the index.
 23 */
 24struct io_factory {
 25	struct block_device *bdev;
 26	atomic_t ref_count;
 27};
 28
 29/* The buffered reader allows efficient I/O by reading page-sized segments into a buffer. */
 30struct buffered_reader {
 31	struct io_factory *factory;
 32	struct dm_bufio_client *client;
 33	struct dm_buffer *buffer;
 34	sector_t limit;
 35	sector_t block_number;
 36	u8 *start;
 37	u8 *end;
 38};
 39
 40#define MAX_READ_AHEAD_BLOCKS 4
 41
 42/*
 43 * The buffered writer allows efficient I/O by buffering writes and committing page-sized segments
 44 * to storage.
 45 */
 46struct buffered_writer {
 47	struct io_factory *factory;
 48	struct dm_bufio_client *client;
 49	struct dm_buffer *buffer;
 50	sector_t limit;
 51	sector_t block_number;
 52	u8 *start;
 53	u8 *end;
 54	int error;
 55};
 56
 57static void uds_get_io_factory(struct io_factory *factory)
 58{
 59	atomic_inc(&factory->ref_count);
 60}
 61
 62int uds_make_io_factory(struct block_device *bdev, struct io_factory **factory_ptr)
 63{
 64	int result;
 65	struct io_factory *factory;
 66
 67	result = vdo_allocate(1, struct io_factory, __func__, &factory);
 68	if (result != VDO_SUCCESS)
 69		return result;
 70
 71	factory->bdev = bdev;
 72	atomic_set_release(&factory->ref_count, 1);
 73
 74	*factory_ptr = factory;
 75	return UDS_SUCCESS;
 76}
 77
 78int uds_replace_storage(struct io_factory *factory, struct block_device *bdev)
 79{
 80	factory->bdev = bdev;
 81	return UDS_SUCCESS;
 82}
 83
 84/* Free an I/O factory once all references have been released. */
 85void uds_put_io_factory(struct io_factory *factory)
 86{
 87	if (atomic_add_return(-1, &factory->ref_count) <= 0)
 88		vdo_free(factory);
 89}
 90
 91size_t uds_get_writable_size(struct io_factory *factory)
 92{
 93	return bdev_nr_bytes(factory->bdev);
 94}
 95
 96/* Create a struct dm_bufio_client for an index region starting at offset. */
 97int uds_make_bufio(struct io_factory *factory, off_t block_offset, size_t block_size,
 98		   unsigned int reserved_buffers, struct dm_bufio_client **client_ptr)
 99{
100	struct dm_bufio_client *client;
101
102	client = dm_bufio_client_create(factory->bdev, block_size, reserved_buffers, 0,
103					NULL, NULL, 0);
104	if (IS_ERR(client))
105		return -PTR_ERR(client);
106
107	dm_bufio_set_sector_offset(client, block_offset * SECTORS_PER_BLOCK);
108	*client_ptr = client;
109	return UDS_SUCCESS;
110}
111
112static void read_ahead(struct buffered_reader *reader, sector_t block_number)
113{
114	if (block_number < reader->limit) {
115		sector_t read_ahead = min((sector_t) MAX_READ_AHEAD_BLOCKS,
116					  reader->limit - block_number);
117
118		dm_bufio_prefetch(reader->client, block_number, read_ahead);
119	}
120}
121
122void uds_free_buffered_reader(struct buffered_reader *reader)
123{
124	if (reader == NULL)
125		return;
126
127	if (reader->buffer != NULL)
128		dm_bufio_release(reader->buffer);
129
130	dm_bufio_client_destroy(reader->client);
131	uds_put_io_factory(reader->factory);
132	vdo_free(reader);
133}
134
135/* Create a buffered reader for an index region starting at offset. */
136int uds_make_buffered_reader(struct io_factory *factory, off_t offset, u64 block_count,
137			     struct buffered_reader **reader_ptr)
138{
139	int result;
140	struct dm_bufio_client *client = NULL;
141	struct buffered_reader *reader = NULL;
142
143	result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);
144	if (result != UDS_SUCCESS)
145		return result;
146
147	result = vdo_allocate(1, struct buffered_reader, "buffered reader", &reader);
148	if (result != VDO_SUCCESS) {
149		dm_bufio_client_destroy(client);
150		return result;
151	}
152
153	*reader = (struct buffered_reader) {
154		.factory = factory,
155		.client = client,
156		.buffer = NULL,
157		.limit = block_count,
158		.block_number = 0,
159		.start = NULL,
160		.end = NULL,
161	};
162
163	read_ahead(reader, 0);
164	uds_get_io_factory(factory);
165	*reader_ptr = reader;
166	return UDS_SUCCESS;
167}
168
169static int position_reader(struct buffered_reader *reader, sector_t block_number,
170			   off_t offset)
171{
172	struct dm_buffer *buffer = NULL;
173	void *data;
174
175	if ((reader->end == NULL) || (block_number != reader->block_number)) {
176		if (block_number >= reader->limit)
177			return UDS_OUT_OF_RANGE;
178
179		if (reader->buffer != NULL)
180			dm_bufio_release(vdo_forget(reader->buffer));
181
182		data = dm_bufio_read(reader->client, block_number, &buffer);
183		if (IS_ERR(data))
184			return -PTR_ERR(data);
185
186		reader->buffer = buffer;
187		reader->start = data;
188		if (block_number == reader->block_number + 1)
189			read_ahead(reader, block_number + 1);
190	}
191
192	reader->block_number = block_number;
193	reader->end = reader->start + offset;
194	return UDS_SUCCESS;
195}
196
197static size_t bytes_remaining_in_read_buffer(struct buffered_reader *reader)
198{
199	return (reader->end == NULL) ? 0 : reader->start + UDS_BLOCK_SIZE - reader->end;
200}
201
202static int reset_reader(struct buffered_reader *reader)
203{
204	sector_t block_number;
205
206	if (bytes_remaining_in_read_buffer(reader) > 0)
207		return UDS_SUCCESS;
208
209	block_number = reader->block_number;
210	if (reader->end != NULL)
211		block_number++;
212
213	return position_reader(reader, block_number, 0);
214}
215
216int uds_read_from_buffered_reader(struct buffered_reader *reader, u8 *data,
217				  size_t length)
218{
219	int result = UDS_SUCCESS;
220	size_t chunk_size;
221
222	while (length > 0) {
223		result = reset_reader(reader);
224		if (result != UDS_SUCCESS)
225			return result;
226
227		chunk_size = min(length, bytes_remaining_in_read_buffer(reader));
228		memcpy(data, reader->end, chunk_size);
229		length -= chunk_size;
230		data += chunk_size;
231		reader->end += chunk_size;
232	}
233
234	return UDS_SUCCESS;
235}
236
237/*
238 * Verify that the next data on the reader matches the required value. If the value matches, the
239 * matching contents are consumed. If the value does not match, the reader state is unchanged.
240 */
241int uds_verify_buffered_data(struct buffered_reader *reader, const u8 *value,
242			     size_t length)
243{
244	int result = UDS_SUCCESS;
245	size_t chunk_size;
246	sector_t start_block_number = reader->block_number;
247	int start_offset = reader->end - reader->start;
248
249	while (length > 0) {
250		result = reset_reader(reader);
251		if (result != UDS_SUCCESS) {
252			result = UDS_CORRUPT_DATA;
253			break;
254		}
255
256		chunk_size = min(length, bytes_remaining_in_read_buffer(reader));
257		if (memcmp(value, reader->end, chunk_size) != 0) {
258			result = UDS_CORRUPT_DATA;
259			break;
260		}
261
262		length -= chunk_size;
263		value += chunk_size;
264		reader->end += chunk_size;
265	}
266
267	if (result != UDS_SUCCESS)
268		position_reader(reader, start_block_number, start_offset);
269
270	return result;
271}
272
273/* Create a buffered writer for an index region starting at offset. */
274int uds_make_buffered_writer(struct io_factory *factory, off_t offset, u64 block_count,
275			     struct buffered_writer **writer_ptr)
276{
277	int result;
278	struct dm_bufio_client *client = NULL;
279	struct buffered_writer *writer;
280
281	result = uds_make_bufio(factory, offset, UDS_BLOCK_SIZE, 1, &client);
282	if (result != UDS_SUCCESS)
283		return result;
284
285	result = vdo_allocate(1, struct buffered_writer, "buffered writer", &writer);
286	if (result != VDO_SUCCESS) {
287		dm_bufio_client_destroy(client);
288		return result;
289	}
290
291	*writer = (struct buffered_writer) {
292		.factory = factory,
293		.client = client,
294		.buffer = NULL,
295		.limit = block_count,
296		.start = NULL,
297		.end = NULL,
298		.block_number = 0,
299		.error = UDS_SUCCESS,
300	};
301
302	uds_get_io_factory(factory);
303	*writer_ptr = writer;
304	return UDS_SUCCESS;
305}
306
307static size_t get_remaining_write_space(struct buffered_writer *writer)
308{
309	return writer->start + UDS_BLOCK_SIZE - writer->end;
310}
311
312static int __must_check prepare_next_buffer(struct buffered_writer *writer)
313{
314	struct dm_buffer *buffer = NULL;
315	void *data;
316
317	if (writer->block_number >= writer->limit) {
318		writer->error = UDS_OUT_OF_RANGE;
319		return UDS_OUT_OF_RANGE;
320	}
321
322	data = dm_bufio_new(writer->client, writer->block_number, &buffer);
323	if (IS_ERR(data)) {
324		writer->error = -PTR_ERR(data);
325		return writer->error;
326	}
327
328	writer->buffer = buffer;
329	writer->start = data;
330	writer->end = data;
331	return UDS_SUCCESS;
332}
333
334static int flush_previous_buffer(struct buffered_writer *writer)
335{
336	size_t available;
337
338	if (writer->buffer == NULL)
339		return writer->error;
340
341	if (writer->error == UDS_SUCCESS) {
342		available = get_remaining_write_space(writer);
343
344		if (available > 0)
345			memset(writer->end, 0, available);
346
347		dm_bufio_mark_buffer_dirty(writer->buffer);
348	}
349
350	dm_bufio_release(writer->buffer);
351	writer->buffer = NULL;
352	writer->start = NULL;
353	writer->end = NULL;
354	writer->block_number++;
355	return writer->error;
356}
357
358void uds_free_buffered_writer(struct buffered_writer *writer)
359{
360	int result;
361
362	if (writer == NULL)
363		return;
364
365	flush_previous_buffer(writer);
366	result = -dm_bufio_write_dirty_buffers(writer->client);
367	if (result != UDS_SUCCESS)
368		vdo_log_warning_strerror(result, "%s: failed to sync storage", __func__);
369
370	dm_bufio_client_destroy(writer->client);
371	uds_put_io_factory(writer->factory);
372	vdo_free(writer);
373}
374
375/*
376 * Append data to the buffer, writing as needed. If no data is provided, zeros are written instead.
377 * If a write error occurs, it is recorded and returned on every subsequent write attempt.
378 */
379int uds_write_to_buffered_writer(struct buffered_writer *writer, const u8 *data,
380				 size_t length)
381{
382	int result = writer->error;
383	size_t chunk_size;
384
385	while ((length > 0) && (result == UDS_SUCCESS)) {
386		if (writer->buffer == NULL) {
387			result = prepare_next_buffer(writer);
388			continue;
389		}
390
391		chunk_size = min(length, get_remaining_write_space(writer));
392		if (data == NULL) {
393			memset(writer->end, 0, chunk_size);
394		} else {
395			memcpy(writer->end, data, chunk_size);
396			data += chunk_size;
397		}
398
399		length -= chunk_size;
400		writer->end += chunk_size;
401
402		if (get_remaining_write_space(writer) == 0)
403			result = uds_flush_buffered_writer(writer);
404	}
405
406	return result;
407}
408
409int uds_flush_buffered_writer(struct buffered_writer *writer)
410{
411	if (writer->error != UDS_SUCCESS)
412		return writer->error;
413
414	return flush_previous_buffer(writer);
415}