instrumented.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2
  3/*
  4 * This header provides generic wrappers for memory access instrumentation that
  5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
  6 */
  7#ifndef _LINUX_INSTRUMENTED_H
  8#define _LINUX_INSTRUMENTED_H
  9
 10#include <linux/compiler.h>
 11#include <linux/kasan-checks.h>
 12#include <linux/kcsan-checks.h>
 13#include <linux/kmsan-checks.h>
 14#include <linux/types.h>
 15
 16/**
 17 * instrument_read - instrument regular read access
 18 * @v: address of access
 19 * @size: size of access
 20 *
 21 * Instrument a regular read access. The instrumentation should be inserted
 22 * before the actual read happens.
 23 */
 24static __always_inline void instrument_read(const volatile void *v, size_t size)
 25{
 26	kasan_check_read(v, size);
 27	kcsan_check_read(v, size);
 28}
 29
 30/**
 31 * instrument_write - instrument regular write access
 32 * @v: address of access
 33 * @size: size of access
 34 *
 35 * Instrument a regular write access. The instrumentation should be inserted
 36 * before the actual write happens.
 37 */
 38static __always_inline void instrument_write(const volatile void *v, size_t size)
 39{
 40	kasan_check_write(v, size);
 41	kcsan_check_write(v, size);
 42}
 43
 44/**
 45 * instrument_read_write - instrument regular read-write access
 46 * @v: address of access
 47 * @size: size of access
 48 *
 49 * Instrument a regular write access. The instrumentation should be inserted
 50 * before the actual write happens.
 51 */
 52static __always_inline void instrument_read_write(const volatile void *v, size_t size)
 53{
 54	kasan_check_write(v, size);
 55	kcsan_check_read_write(v, size);
 56}
 57
 58/**
 59 * instrument_atomic_read - instrument atomic read access
 60 * @v: address of access
 61 * @size: size of access
 62 *
 63 * Instrument an atomic read access. The instrumentation should be inserted
 64 * before the actual read happens.
 65 */
 66static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
 67{
 68	kasan_check_read(v, size);
 69	kcsan_check_atomic_read(v, size);
 70}
 71
 72/**
 73 * instrument_atomic_write - instrument atomic write access
 74 * @v: address of access
 75 * @size: size of access
 76 *
 77 * Instrument an atomic write access. The instrumentation should be inserted
 78 * before the actual write happens.
 79 */
 80static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
 81{
 82	kasan_check_write(v, size);
 83	kcsan_check_atomic_write(v, size);
 84}
 85
 86/**
 87 * instrument_atomic_read_write - instrument atomic read-write access
 88 * @v: address of access
 89 * @size: size of access
 90 *
 91 * Instrument an atomic read-write access. The instrumentation should be
 92 * inserted before the actual write happens.
 93 */
 94static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
 95{
 96	kasan_check_write(v, size);
 97	kcsan_check_atomic_read_write(v, size);
 98}
 99
100/**
101 * instrument_copy_to_user - instrument reads of copy_to_user
102 * @to: destination address
103 * @from: source address
104 * @n: number of bytes to copy
105 *
106 * Instrument reads from kernel memory, that are due to copy_to_user (and
107 * variants). The instrumentation must be inserted before the accesses.
108 */
109static __always_inline void
110instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
111{
112	kasan_check_read(from, n);
113	kcsan_check_read(from, n);
114	kmsan_copy_to_user(to, from, n, 0);
115}
116
117/**
118 * instrument_copy_from_user_before - add instrumentation before copy_from_user
119 * @to: destination address
120 * @from: source address
121 * @n: number of bytes to copy
122 *
123 * Instrument writes to kernel memory, that are due to copy_from_user (and
124 * variants). The instrumentation should be inserted before the accesses.
125 */
126static __always_inline void
127instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
128{
129	kasan_check_write(to, n);
130	kcsan_check_write(to, n);
131}
132
133/**
134 * instrument_copy_from_user_after - add instrumentation after copy_from_user
135 * @to: destination address
136 * @from: source address
137 * @n: number of bytes to copy
138 * @left: number of bytes not copied (as returned by copy_from_user)
139 *
140 * Instrument writes to kernel memory, that are due to copy_from_user (and
141 * variants). The instrumentation should be inserted after the accesses.
142 */
143static __always_inline void
144instrument_copy_from_user_after(const void *to, const void __user *from,
145				unsigned long n, unsigned long left)
146{
147	kmsan_unpoison_memory(to, n - left);
148}
149
150/**
151 * instrument_memcpy_before - add instrumentation before non-instrumented memcpy
152 * @to: destination address
153 * @from: source address
154 * @n: number of bytes to copy
155 *
156 * Instrument memory accesses that happen in custom memcpy implementations. The
157 * instrumentation should be inserted before the memcpy call.
158 */
159static __always_inline void instrument_memcpy_before(void *to, const void *from,
160						     unsigned long n)
161{
162	kasan_check_write(to, n);
163	kasan_check_read(from, n);
164	kcsan_check_write(to, n);
165	kcsan_check_read(from, n);
166}
167
168/**
169 * instrument_memcpy_after - add instrumentation after non-instrumented memcpy
170 * @to: destination address
171 * @from: source address
172 * @n: number of bytes to copy
173 * @left: number of bytes not copied (if known)
174 *
175 * Instrument memory accesses that happen in custom memcpy implementations. The
176 * instrumentation should be inserted after the memcpy call.
177 */
178static __always_inline void instrument_memcpy_after(void *to, const void *from,
179						    unsigned long n,
180						    unsigned long left)
181{
182	kmsan_memmove(to, from, n - left);
183}
184
185/**
186 * instrument_get_user() - add instrumentation to get_user()-like macros
187 * @to: destination variable, may not be address-taken
188 *
189 * get_user() and friends are fragile, so it may depend on the implementation
190 * whether the instrumentation happens before or after the data is copied from
191 * the userspace.
192 */
193#define instrument_get_user(to)				\
194({							\
195	u64 __tmp = (u64)(to);				\
196	kmsan_unpoison_memory(&__tmp, sizeof(__tmp));	\
197	to = __tmp;					\
198})
199
200
201/**
202 * instrument_put_user() - add instrumentation to put_user()-like macros
203 * @from: source address
204 * @ptr: userspace pointer to copy to
205 * @size: number of bytes to copy
206 *
207 * put_user() and friends are fragile, so it may depend on the implementation
208 * whether the instrumentation happens before or after the data is copied from
209 * the userspace.
210 */
211#define instrument_put_user(from, ptr, size)			\
212({								\
213	kmsan_copy_to_user(ptr, &from, sizeof(from), 0);	\
214})
215
216#endif /* _LINUX_INSTRUMENTED_H */