logical-zone.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright 2023 Red Hat
  4 */
  5
  6#include "logical-zone.h"
  7
  8#include "logger.h"
  9#include "memory-alloc.h"
 10#include "permassert.h"
 11#include "string-utils.h"
 12
 13#include "action-manager.h"
 14#include "admin-state.h"
 15#include "block-map.h"
 16#include "completion.h"
 17#include "constants.h"
 18#include "data-vio.h"
 19#include "flush.h"
 20#include "int-map.h"
 21#include "physical-zone.h"
 22#include "vdo.h"
 23
 24#define ALLOCATIONS_PER_ZONE 128
 25
 26/**
 27 * as_logical_zone() - Convert a generic vdo_completion to a logical_zone.
 28 * @completion: The completion to convert.
 29 *
 30 * Return: The completion as a logical_zone.
 31 */
 32static struct logical_zone *as_logical_zone(struct vdo_completion *completion)
 33{
 34	vdo_assert_completion_type(completion, VDO_GENERATION_FLUSHED_COMPLETION);
 35	return container_of(completion, struct logical_zone, completion);
 36}
 37
 38/* get_thread_id_for_zone() - Implements vdo_zone_thread_getter_fn. */
 39static thread_id_t get_thread_id_for_zone(void *context, zone_count_t zone_number)
 40{
 41	struct logical_zones *zones = context;
 42
 43	return zones->zones[zone_number].thread_id;
 44}
 45
 46/**
 47 * initialize_zone() - Initialize a logical zone.
 48 * @zones: The logical_zones to which this zone belongs.
 49 * @zone_number: The logical_zone's index.
 50 */
 51static int initialize_zone(struct logical_zones *zones, zone_count_t zone_number)
 52{
 53	int result;
 54	struct vdo *vdo = zones->vdo;
 55	struct logical_zone *zone = &zones->zones[zone_number];
 56	zone_count_t allocation_zone_number;
 57
 58	result = vdo_int_map_create(VDO_LOCK_MAP_CAPACITY, &zone->lbn_operations);
 59	if (result != VDO_SUCCESS)
 60		return result;
 61
 62	if (zone_number < vdo->thread_config.logical_zone_count - 1)
 63		zone->next = &zones->zones[zone_number + 1];
 64
 65	vdo_initialize_completion(&zone->completion, vdo,
 66				  VDO_GENERATION_FLUSHED_COMPLETION);
 67	zone->zones = zones;
 68	zone->zone_number = zone_number;
 69	zone->thread_id = vdo->thread_config.logical_threads[zone_number];
 70	zone->block_map_zone = &vdo->block_map->zones[zone_number];
 71	INIT_LIST_HEAD(&zone->write_vios);
 72	vdo_set_admin_state_code(&zone->state, VDO_ADMIN_STATE_NORMAL_OPERATION);
 73
 74	allocation_zone_number = zone->thread_id % vdo->thread_config.physical_zone_count;
 75	zone->allocation_zone = &vdo->physical_zones->zones[allocation_zone_number];
 76
 77	return vdo_make_default_thread(vdo, zone->thread_id);
 78}
 79
 80/**
 81 * vdo_make_logical_zones() - Create a set of logical zones.
 82 * @vdo: The vdo to which the zones will belong.
 83 * @zones_ptr: A pointer to hold the new zones.
 84 *
 85 * Return: VDO_SUCCESS or an error code.
 86 */
 87int vdo_make_logical_zones(struct vdo *vdo, struct logical_zones **zones_ptr)
 88{
 89	struct logical_zones *zones;
 90	int result;
 91	zone_count_t zone;
 92	zone_count_t zone_count = vdo->thread_config.logical_zone_count;
 93
 94	if (zone_count == 0)
 95		return VDO_SUCCESS;
 96
 97	result = vdo_allocate_extended(struct logical_zones, zone_count,
 98				       struct logical_zone, __func__, &zones);
 99	if (result != VDO_SUCCESS)
100		return result;
101
102	zones->vdo = vdo;
103	zones->zone_count = zone_count;
104	for (zone = 0; zone < zone_count; zone++) {
105		result = initialize_zone(zones, zone);
106		if (result != VDO_SUCCESS) {
107			vdo_free_logical_zones(zones);
108			return result;
109		}
110	}
111
112	result = vdo_make_action_manager(zones->zone_count, get_thread_id_for_zone,
113					 vdo->thread_config.admin_thread, zones, NULL,
114					 vdo, &zones->manager);
115	if (result != VDO_SUCCESS) {
116		vdo_free_logical_zones(zones);
117		return result;
118	}
119
120	*zones_ptr = zones;
121	return VDO_SUCCESS;
122}
123
124/**
125 * vdo_free_logical_zones() - Free a set of logical zones.
126 * @zones: The set of zones to free.
127 */
128void vdo_free_logical_zones(struct logical_zones *zones)
129{
130	zone_count_t index;
131
132	if (zones == NULL)
133		return;
134
135	vdo_free(vdo_forget(zones->manager));
136
137	for (index = 0; index < zones->zone_count; index++)
138		vdo_int_map_free(vdo_forget(zones->zones[index].lbn_operations));
139
140	vdo_free(zones);
141}
142
143static inline void assert_on_zone_thread(struct logical_zone *zone, const char *what)
144{
145	VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == zone->thread_id),
146			    "%s() called on correct thread", what);
147}
148
149/**
150 * check_for_drain_complete() - Check whether this zone has drained.
151 * @zone: The zone to check.
152 */
153static void check_for_drain_complete(struct logical_zone *zone)
154{
155	if (!vdo_is_state_draining(&zone->state) || zone->notifying ||
156	    !list_empty(&zone->write_vios))
157		return;
158
159	vdo_finish_draining(&zone->state);
160}
161
162/** Implements vdo_admin_initiator_fn. */
163static void initiate_drain(struct admin_state *state)
164{
165	check_for_drain_complete(container_of(state, struct logical_zone, state));
166}
167
168/** Implements vdo_zone_action_fn. */
169static void drain_logical_zone(void *context, zone_count_t zone_number,
170			       struct vdo_completion *parent)
171{
172	struct logical_zones *zones = context;
173
174	vdo_start_draining(&zones->zones[zone_number].state,
175			   vdo_get_current_manager_operation(zones->manager), parent,
176			   initiate_drain);
177}
178
179void vdo_drain_logical_zones(struct logical_zones *zones,
180			     const struct admin_state_code *operation,
181			     struct vdo_completion *parent)
182{
183	vdo_schedule_operation(zones->manager, operation, NULL, drain_logical_zone, NULL,
184			       parent);
185}
186
187/** Implements vdo_zone_action_fn. */
188static void resume_logical_zone(void *context, zone_count_t zone_number,
189				struct vdo_completion *parent)
190{
191	struct logical_zone *zone = &(((struct logical_zones *) context)->zones[zone_number]);
192
193	vdo_fail_completion(parent, vdo_resume_if_quiescent(&zone->state));
194}
195
196/**
197 * vdo_resume_logical_zones() - Resume a set of logical zones.
198 * @zones: The logical zones to resume.
199 * @parent: The object to notify when the zones have resumed.
200 */
201void vdo_resume_logical_zones(struct logical_zones *zones, struct vdo_completion *parent)
202{
203	vdo_schedule_operation(zones->manager, VDO_ADMIN_STATE_RESUMING, NULL,
204			       resume_logical_zone, NULL, parent);
205}
206
207/**
208 * update_oldest_active_generation() - Update the oldest active generation.
209 * @zone: The zone.
210 *
211 * Return: true if the oldest active generation has changed.
212 */
213static bool update_oldest_active_generation(struct logical_zone *zone)
214{
215	struct data_vio *data_vio =
216		list_first_entry_or_null(&zone->write_vios, struct data_vio,
217					 write_entry);
218	sequence_number_t oldest =
219		(data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation;
220
221	if (oldest == zone->oldest_active_generation)
222		return false;
223
224	WRITE_ONCE(zone->oldest_active_generation, oldest);
225	return true;
226}
227
228/**
229 * vdo_increment_logical_zone_flush_generation() - Increment the flush generation in a logical
230 *                                                 zone.
231 * @zone: The logical zone.
232 * @expected_generation: The expected value of the flush generation before the increment.
233 */
234void vdo_increment_logical_zone_flush_generation(struct logical_zone *zone,
235						 sequence_number_t expected_generation)
236{
237	assert_on_zone_thread(zone, __func__);
238	VDO_ASSERT_LOG_ONLY((zone->flush_generation == expected_generation),
239			    "logical zone %u flush generation %llu should be %llu before increment",
240			    zone->zone_number, (unsigned long long) zone->flush_generation,
241			    (unsigned long long) expected_generation);
242
243	zone->flush_generation++;
244	zone->ios_in_flush_generation = 0;
245	update_oldest_active_generation(zone);
246}
247
248/**
249 * vdo_acquire_flush_generation_lock() - Acquire the shared lock on a flush generation by a write
250 *                                       data_vio.
251 * @data_vio: The data_vio.
252 */
253void vdo_acquire_flush_generation_lock(struct data_vio *data_vio)
254{
255	struct logical_zone *zone = data_vio->logical.zone;
256
257	assert_on_zone_thread(zone, __func__);
258	VDO_ASSERT_LOG_ONLY(vdo_is_state_normal(&zone->state), "vdo state is normal");
259
260	data_vio->flush_generation = zone->flush_generation;
261	list_add_tail(&data_vio->write_entry, &zone->write_vios);
262	zone->ios_in_flush_generation++;
263}
264
265static void attempt_generation_complete_notification(struct vdo_completion *completion);
266
267/**
268 * notify_flusher() - Notify the flush that at least one generation no longer has active VIOs.
269 * @completion: The zone completion.
270 *
271 * This callback is registered in attempt_generation_complete_notification().
272 */
273static void notify_flusher(struct vdo_completion *completion)
274{
275	struct logical_zone *zone = as_logical_zone(completion);
276
277	vdo_complete_flushes(zone->zones->vdo->flusher);
278	vdo_launch_completion_callback(completion,
279				       attempt_generation_complete_notification,
280				       zone->thread_id);
281}
282
283/**
284 * attempt_generation_complete_notification() - Notify the flusher if some generation no
285 *                                              longer has active VIOs.
286 * @completion: The zone completion.
287 */
288static void attempt_generation_complete_notification(struct vdo_completion *completion)
289{
290	struct logical_zone *zone = as_logical_zone(completion);
291
292	assert_on_zone_thread(zone, __func__);
293	if (zone->oldest_active_generation <= zone->notification_generation) {
294		zone->notifying = false;
295		check_for_drain_complete(zone);
296		return;
297	}
298
299	zone->notifying = true;
300	zone->notification_generation = zone->oldest_active_generation;
301	vdo_launch_completion_callback(&zone->completion, notify_flusher,
302				       vdo_get_flusher_thread_id(zone->zones->vdo->flusher));
303}
304
305/**
306 * vdo_release_flush_generation_lock() - Release the shared lock on a flush generation held by a
307 *                                       write data_vio.
308 * @data_vio: The data_vio whose lock is to be released.
309 *
310 * If there are pending flushes, and this data_vio completes the oldest generation active in this
311 * zone, an attempt will be made to finish any flushes which may now be complete.
312 */
313void vdo_release_flush_generation_lock(struct data_vio *data_vio)
314{
315	struct logical_zone *zone = data_vio->logical.zone;
316
317	assert_on_zone_thread(zone, __func__);
318
319	if (!data_vio_has_flush_generation_lock(data_vio))
320		return;
321
322	list_del_init(&data_vio->write_entry);
323	VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation),
324			    "data_vio releasing lock on generation %llu is not older than oldest active generation %llu",
325			    (unsigned long long) data_vio->flush_generation,
326			    (unsigned long long) zone->oldest_active_generation);
327
328	if (!update_oldest_active_generation(zone) || zone->notifying)
329		return;
330
331	attempt_generation_complete_notification(&zone->completion);
332}
333
334struct physical_zone *vdo_get_next_allocation_zone(struct logical_zone *zone)
335{
336	if (zone->allocation_count == ALLOCATIONS_PER_ZONE) {
337		zone->allocation_count = 0;
338		zone->allocation_zone = zone->allocation_zone->next;
339	}
340
341	zone->allocation_count++;
342	return zone->allocation_zone;
343}
344
345/**
346 * vdo_dump_logical_zone() - Dump information about a logical zone to the log for debugging.
347 * @zone: The zone to dump.
348 *
349 * Context: the information is dumped in a thread-unsafe fashion.
350 *
351 */
352void vdo_dump_logical_zone(const struct logical_zone *zone)
353{
354	vdo_log_info("logical_zone %u", zone->zone_number);
355	vdo_log_info("  flush_generation=%llu oldest_active_generation=%llu notification_generation=%llu notifying=%s ios_in_flush_generation=%llu",
356		     (unsigned long long) READ_ONCE(zone->flush_generation),
357		     (unsigned long long) READ_ONCE(zone->oldest_active_generation),
358		     (unsigned long long) READ_ONCE(zone->notification_generation),
359		     vdo_bool_to_string(READ_ONCE(zone->notifying)),
360		     (unsigned long long) READ_ONCE(zone->ios_in_flush_generation));
361}