vdo.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright 2023 Red Hat
  4 */
  5
  6#ifndef VDO_H
  7#define VDO_H
  8
  9#include <linux/atomic.h>
 10#include <linux/blk_types.h>
 11#include <linux/completion.h>
 12#include <linux/dm-kcopyd.h>
 13#include <linux/list.h>
 14#include <linux/spinlock.h>
 15
 16#include "admin-state.h"
 17#include "encodings.h"
 18#include "funnel-workqueue.h"
 19#include "packer.h"
 20#include "physical-zone.h"
 21#include "statistics.h"
 22#include "thread-registry.h"
 23#include "types.h"
 24
 25enum notifier_state {
 26	/* Notifications are allowed but not in progress */
 27	MAY_NOTIFY,
 28	/* A notification is in progress */
 29	NOTIFYING,
 30	/* Notifications are not allowed */
 31	MAY_NOT_NOTIFY,
 32	/* A notification has completed */
 33	NOTIFIED,
 34};
 35
 36/**
 37 * typedef vdo_read_only_notification_fn - A function to notify a listener that the VDO has gone
 38 *                                         read-only.
 39 * @listener: The object to notify.
 40 * @parent: The completion to notify in order to acknowledge the notification.
 41 */
 42typedef void (*vdo_read_only_notification_fn)(void *listener, struct vdo_completion *parent);
 43
 44/*
 45 * An object to be notified when the VDO enters read-only mode
 46 */
 47struct read_only_listener {
 48	/* The listener */
 49	void *listener;
 50	/* The method to call to notify the listener */
 51	vdo_read_only_notification_fn notify;
 52	/* A pointer to the next listener */
 53	struct read_only_listener *next;
 54};
 55
 56struct vdo_thread {
 57	struct vdo *vdo;
 58	thread_id_t thread_id;
 59	struct vdo_work_queue *queue;
 60	/*
 61	 * Each thread maintains its own notion of whether the VDO is read-only so that the
 62	 * read-only state can be checked from any base thread without worrying about
 63	 * synchronization or thread safety. This does mean that knowledge of the VDO going
 64	 * read-only does not occur simultaneously across the VDO's threads, but that does not seem
 65	 * to cause any problems.
 66	 */
 67	bool is_read_only;
 68	/*
 69	 * A list of objects waiting to be notified on this thread that the VDO has entered
 70	 * read-only mode.
 71	 */
 72	struct read_only_listener *listeners;
 73	struct registered_thread allocating_thread;
 74};
 75
 76/* Keep struct bio statistics atomically */
 77struct atomic_bio_stats {
 78	atomic64_t read; /* Number of not REQ_WRITE bios */
 79	atomic64_t write; /* Number of REQ_WRITE bios */
 80	atomic64_t discard; /* Number of REQ_DISCARD bios */
 81	atomic64_t flush; /* Number of REQ_FLUSH bios */
 82	atomic64_t empty_flush; /* Number of REQ_PREFLUSH bios without data */
 83	atomic64_t fua; /* Number of REQ_FUA bios */
 84};
 85
 86/* Counters are atomic since updates can arrive concurrently from arbitrary threads. */
 87struct atomic_statistics {
 88	atomic64_t bios_submitted;
 89	atomic64_t bios_completed;
 90	atomic64_t flush_out;
 91	atomic64_t invalid_advice_pbn_count;
 92	atomic64_t no_space_error_count;
 93	atomic64_t read_only_error_count;
 94	struct atomic_bio_stats bios_in;
 95	struct atomic_bio_stats bios_in_partial;
 96	struct atomic_bio_stats bios_out;
 97	struct atomic_bio_stats bios_out_completed;
 98	struct atomic_bio_stats bios_acknowledged;
 99	struct atomic_bio_stats bios_acknowledged_partial;
100	struct atomic_bio_stats bios_meta;
101	struct atomic_bio_stats bios_meta_completed;
102	struct atomic_bio_stats bios_journal;
103	struct atomic_bio_stats bios_journal_completed;
104	struct atomic_bio_stats bios_page_cache;
105	struct atomic_bio_stats bios_page_cache_completed;
106};
107
108struct read_only_notifier {
109	/* The completion for entering read-only mode */
110	struct vdo_completion completion;
111	/* A completion waiting for notifications to be drained or enabled */
112	struct vdo_completion *waiter;
113	/* Lock to protect the next two fields */
114	spinlock_t lock;
115	/* The code of the error which put the VDO into read-only mode */
116	int read_only_error;
117	/* The current state of the notifier (values described above) */
118	enum notifier_state state;
119};
120
121/*
122 * The thread ID returned when the current thread is not a vdo thread, or can not be determined
123 * (usually due to being at interrupt context).
124 */
125#define VDO_INVALID_THREAD_ID ((thread_id_t) -1)
126
127struct thread_config {
128	zone_count_t logical_zone_count;
129	zone_count_t physical_zone_count;
130	zone_count_t hash_zone_count;
131	thread_count_t bio_thread_count;
132	thread_count_t thread_count;
133	thread_id_t admin_thread;
134	thread_id_t journal_thread;
135	thread_id_t packer_thread;
136	thread_id_t dedupe_thread;
137	thread_id_t bio_ack_thread;
138	thread_id_t cpu_thread;
139	thread_id_t *logical_threads;
140	thread_id_t *physical_threads;
141	thread_id_t *hash_zone_threads;
142	thread_id_t *bio_threads;
143};
144
145struct thread_count_config;
146
147struct vdo_super_block {
148	/* The vio for reading and writing the super block to disk */
149	struct vio vio;
150	/* A buffer to hold the super block */
151	u8 *buffer;
152	/* Whether this super block may not be written */
153	bool unwritable;
154};
155
156struct data_vio_pool;
157
158struct vdo_administrator {
159	struct vdo_completion completion;
160	struct admin_state state;
161	atomic_t busy;
162	u32 phase;
163	struct completion callback_sync;
164};
165
166struct vdo {
167	char thread_name_prefix[MAX_VDO_WORK_QUEUE_NAME_LEN];
168	struct vdo_thread *threads;
169	vdo_action_fn action;
170	struct vdo_completion *completion;
171	struct vio_tracer *vio_tracer;
172
173	/* The atomic version of the state of this vdo */
174	atomic_t state;
175	/* The full state of all components */
176	struct vdo_component_states states;
177	/*
178	 * A counter value to attach to thread names and log messages to identify the individual
179	 * device.
180	 */
181	unsigned int instance;
182	/* The read-only notifier */
183	struct read_only_notifier read_only_notifier;
184	/* The load-time configuration of this vdo */
185	struct device_config *device_config;
186	/* The thread mapping */
187	struct thread_config thread_config;
188
189	/* The super block */
190	struct vdo_super_block super_block;
191
192	/* The partitioning of the underlying storage */
193	struct layout layout;
194	struct layout next_layout;
195	struct dm_kcopyd_client *partition_copier;
196
197	/* The block map */
198	struct block_map *block_map;
199
200	/* The journal for block map recovery */
201	struct recovery_journal *recovery_journal;
202
203	/* The slab depot */
204	struct slab_depot *depot;
205
206	/* The compressed-block packer */
207	struct packer *packer;
208	/* Whether incoming data should be compressed */
209	bool compressing;
210
211	/* The handler for flush requests */
212	struct flusher *flusher;
213
214	/* The state the vdo was in when loaded (primarily for unit tests) */
215	enum vdo_state load_state;
216
217	/* The logical zones of this vdo */
218	struct logical_zones *logical_zones;
219
220	/* The physical zones of this vdo */
221	struct physical_zones *physical_zones;
222
223	/* The hash lock zones of this vdo */
224	struct hash_zones *hash_zones;
225
226	/* Bio submission manager used for sending bios to the storage device. */
227	struct io_submitter *io_submitter;
228
229	/* The pool of data_vios for servicing incoming bios */
230	struct data_vio_pool *data_vio_pool;
231
232	/* The manager for administrative operations */
233	struct vdo_administrator admin;
234
235	/* Flags controlling administrative operations */
236	const struct admin_state_code *suspend_type;
237	bool allocations_allowed;
238	bool dump_on_shutdown;
239	atomic_t processing_message;
240
241	/*
242	 * Statistics
243	 * Atomic stats counters
244	 */
245	struct atomic_statistics stats;
246	/* Used to gather statistics without allocating memory */
247	struct vdo_statistics stats_buffer;
248	/* Protects the stats_buffer */
249	struct mutex stats_mutex;
250
251	/* A list of all device_configs referencing this vdo */
252	struct list_head device_config_list;
253
254	/* This VDO's list entry for the device registry */
255	struct list_head registration;
256
257	/* Underlying block device info. */
258	u64 starting_sector_offset;
259	struct volume_geometry geometry;
260
261	/* N blobs of context data for LZ4 code, one per CPU thread. */
262	char **compression_context;
263};
264
265/**
266 * vdo_uses_bio_ack_queue() - Indicate whether the vdo is configured to use a separate work queue
267 *                            for acknowledging received and processed bios.
268 * @vdo: The vdo.
269 *
270 * Note that this directly controls the handling of write operations, but the compile-time flag
271 * VDO_USE_BIO_ACK_QUEUE_FOR_READ is also checked for read operations.
272 *
273 * Return: Whether a bio-acknowledgement work queue is in use.
274 */
275static inline bool vdo_uses_bio_ack_queue(struct vdo *vdo)
276{
277	return vdo->device_config->thread_counts.bio_ack_threads > 0;
278}
279
280/**
281 * typedef vdo_filter_fn - Method type for vdo matching methods.
282 * @vdo: The vdo to match.
283 * @context: A parameter for the filter to use.
284 *
285 * Return: True if the vdo matches the filter criteria, false if it doesn't.
286 */
287typedef bool (*vdo_filter_fn)(struct vdo *vdo, const void *context);
288
289void vdo_initialize_device_registry_once(void);
290struct vdo * __must_check vdo_find_matching(vdo_filter_fn filter, const void *context);
291
292int __must_check vdo_make_thread(struct vdo *vdo, thread_id_t thread_id,
293				 const struct vdo_work_queue_type *type,
294				 unsigned int queue_count, void *contexts[]);
295
296static inline int __must_check vdo_make_default_thread(struct vdo *vdo,
297						       thread_id_t thread_id)
298{
299	return vdo_make_thread(vdo, thread_id, NULL, 1, NULL);
300}
301
302int __must_check vdo_make(unsigned int instance, struct device_config *config,
303			  char **reason, struct vdo **vdo_ptr);
304
305void vdo_destroy(struct vdo *vdo);
306
307void vdo_load_super_block(struct vdo *vdo, struct vdo_completion *parent);
308
309struct block_device * __must_check vdo_get_backing_device(const struct vdo *vdo);
310
311const char * __must_check vdo_get_device_name(const struct dm_target *target);
312
313int __must_check vdo_synchronous_flush(struct vdo *vdo);
314
315const struct admin_state_code * __must_check vdo_get_admin_state(const struct vdo *vdo);
316
317bool vdo_set_compressing(struct vdo *vdo, bool enable);
318
319bool vdo_get_compressing(struct vdo *vdo);
320
321void vdo_fetch_statistics(struct vdo *vdo, struct vdo_statistics *stats);
322
323thread_id_t vdo_get_callback_thread_id(void);
324
325enum vdo_state __must_check vdo_get_state(const struct vdo *vdo);
326
327void vdo_set_state(struct vdo *vdo, enum vdo_state state);
328
329void vdo_save_components(struct vdo *vdo, struct vdo_completion *parent);
330
331int vdo_register_read_only_listener(struct vdo *vdo, void *listener,
332				    vdo_read_only_notification_fn notification,
333				    thread_id_t thread_id);
334
335int vdo_enable_read_only_entry(struct vdo *vdo);
336
337void vdo_wait_until_not_entering_read_only_mode(struct vdo_completion *parent);
338
339void vdo_allow_read_only_mode_entry(struct vdo_completion *parent);
340
341void vdo_enter_read_only_mode(struct vdo *vdo, int error_code);
342
343bool __must_check vdo_is_read_only(struct vdo *vdo);
344
345bool __must_check vdo_in_read_only_mode(const struct vdo *vdo);
346
347bool __must_check vdo_in_recovery_mode(const struct vdo *vdo);
348
349void vdo_enter_recovery_mode(struct vdo *vdo);
350
351void vdo_assert_on_admin_thread(const struct vdo *vdo, const char *name);
352
353void vdo_assert_on_logical_zone_thread(const struct vdo *vdo, zone_count_t logical_zone,
354				       const char *name);
355
356void vdo_assert_on_physical_zone_thread(const struct vdo *vdo, zone_count_t physical_zone,
357					const char *name);
358
359int __must_check vdo_get_physical_zone(const struct vdo *vdo, physical_block_number_t pbn,
360				       struct physical_zone **zone_ptr);
361
362void vdo_dump_status(const struct vdo *vdo);
363
364#endif /* VDO_H */