completion.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright 2023 Red Hat
  4 */
  5
  6#include "completion.h"
  7
  8#include <linux/kernel.h>
  9
 10#include "logger.h"
 11#include "permassert.h"
 12
 13#include "status-codes.h"
 14#include "types.h"
 15#include "vio.h"
 16#include "vdo.h"
 17
 18/**
 19 * DOC: vdo completions.
 20 *
 21 * Most of vdo's data structures are lock free, each either belonging to a single "zone," or
 22 * divided into a number of zones whose accesses to the structure do not overlap. During normal
 23 * operation, at most one thread will be operating in any given zone. Each zone has a
 24 * vdo_work_queue which holds vdo_completions that are to be run in that zone. A completion may
 25 * only be enqueued on one queue or operating in a single zone at a time.
 26 *
 27 * At each step of a multi-threaded operation, the completion performing the operation is given a
 28 * callback, error handler, and thread id for the next step. A completion is "run" when it is
 29 * operating on the correct thread (as specified by its callback_thread_id). If the value of its
 30 * "result" field is an error (i.e. not VDO_SUCCESS), the function in its "error_handler" will be
 31 * invoked. If the error_handler is NULL, or there is no error, the function set as its "callback"
 32 * will be invoked. Generally, a completion will not be run directly, but rather will be
 33 * "launched." In this case, it will check whether it is operating on the correct thread. If it is,
 34 * it will run immediately. Otherwise, it will be enqueue on the vdo_work_queue associated with the
 35 * completion's "callback_thread_id". When it is dequeued, it will be on the correct thread, and
 36 * will get run. In some cases, the completion should get queued instead of running immediately,
 37 * even if it is being launched from the correct thread. This is usually in cases where there is a
 38 * long chain of callbacks, all on the same thread, which could overflow the stack. In such cases,
 39 * the completion's "requeue" field should be set to true. Doing so will skip the current thread
 40 * check and simply enqueue the completion.
 41 *
 42 * A completion may be "finished," in which case its "complete" field will be set to true before it
 43 * is next run. It is a bug to attempt to set the result or re-finish a finished completion.
 44 * Because a completion's fields are not safe to examine from any thread other than the one on
 45 * which the completion is currently operating, this field is used only to aid in detecting
 46 * programming errors. It can not be used for cross-thread checking on the status of an operation.
 47 * A completion must be "reset" before it can be reused after it has been finished. Resetting will
 48 * also clear any error from the result field.
 49 **/
 50
 51void vdo_initialize_completion(struct vdo_completion *completion,
 52			       struct vdo *vdo,
 53			       enum vdo_completion_type type)
 54{
 55	memset(completion, 0, sizeof(*completion));
 56	completion->vdo = vdo;
 57	completion->type = type;
 58	vdo_reset_completion(completion);
 59}
 60
 61static inline void assert_incomplete(struct vdo_completion *completion)
 62{
 63	VDO_ASSERT_LOG_ONLY(!completion->complete, "completion is not complete");
 64}
 65
 66/**
 67 * vdo_set_completion_result() - Set the result of a completion.
 68 * @completion: The completion to update.
 69 * @result: The result to set.
 70 *
 71 * Older errors will not be masked.
 72 */
 73void vdo_set_completion_result(struct vdo_completion *completion, int result)
 74{
 75	assert_incomplete(completion);
 76	if (completion->result == VDO_SUCCESS)
 77		completion->result = result;
 78}
 79
 80/**
 81 * vdo_launch_completion_with_priority() - Run or enqueue a completion.
 82 * @completion: The completion to launch.
 83 * @priority: The priority at which to enqueue the completion.
 84 *
 85 * If called on the correct thread (i.e. the one specified in the completion's callback_thread_id
 86 * field) and not marked for requeue, the completion will be run immediately. Otherwise, the
 87 * completion will be enqueued on the specified thread.
 88 */
 89void vdo_launch_completion_with_priority(struct vdo_completion *completion,
 90					 enum vdo_completion_priority priority)
 91{
 92	thread_id_t callback_thread = completion->callback_thread_id;
 93
 94	if (completion->requeue || (callback_thread != vdo_get_callback_thread_id())) {
 95		vdo_enqueue_completion(completion, priority);
 96		return;
 97	}
 98
 99	vdo_run_completion(completion);
100}
101
102/** vdo_finish_completion() - Mark a completion as complete and then launch it. */
103void vdo_finish_completion(struct vdo_completion *completion)
104{
105	assert_incomplete(completion);
106	completion->complete = true;
107	if (completion->callback != NULL)
108		vdo_launch_completion(completion);
109}
110
111void vdo_enqueue_completion(struct vdo_completion *completion,
112			    enum vdo_completion_priority priority)
113{
114	struct vdo *vdo = completion->vdo;
115	thread_id_t thread_id = completion->callback_thread_id;
116
117	if (VDO_ASSERT(thread_id < vdo->thread_config.thread_count,
118		       "thread_id %u (completion type %d) is less than thread count %u",
119		       thread_id, completion->type,
120		       vdo->thread_config.thread_count) != VDO_SUCCESS)
121		BUG();
122
123	completion->requeue = false;
124	completion->priority = priority;
125	completion->my_queue = NULL;
126	vdo_enqueue_work_queue(vdo->threads[thread_id].queue, completion);
127}
128
129/**
130 * vdo_requeue_completion_if_needed() - Requeue a completion if not called on the specified thread.
131 * @completion: The completion to requeue.
132 * @callback_thread_id: The thread on which to requeue the completion.
133 *
134 * Return: True if the completion was requeued; callers may not access the completion in this case.
135 */
136bool vdo_requeue_completion_if_needed(struct vdo_completion *completion,
137				      thread_id_t callback_thread_id)
138{
139	if (vdo_get_callback_thread_id() == callback_thread_id)
140		return false;
141
142	completion->callback_thread_id = callback_thread_id;
143	vdo_enqueue_completion(completion, VDO_WORK_Q_DEFAULT_PRIORITY);
144	return true;
145}