rseq.h

  1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
  2/*
  3 * rseq.h
  4 *
  5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  6 */
  7
  8#ifndef RSEQ_H
  9#define RSEQ_H
 10
 11#include <stdint.h>
 12#include <stdbool.h>
 13#include <pthread.h>
 14#include <signal.h>
 15#include <sched.h>
 16#include <errno.h>
 17#include <stdio.h>
 18#include <stdlib.h>
 19#include <stddef.h>
 20#include "rseq-abi.h"
 21#include "compiler.h"
 22
 23#ifndef rseq_sizeof_field
 24#define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
 25#endif
 26
 27#ifndef rseq_offsetofend
 28#define rseq_offsetofend(TYPE, MEMBER) \
 29	(offsetof(TYPE, MEMBER)	+ rseq_sizeof_field(TYPE, MEMBER))
 30#endif
 31
 32/*
 33 * Empty code injection macros, override when testing.
 34 * It is important to consider that the ASM injection macros need to be
 35 * fully reentrant (e.g. do not modify the stack).
 36 */
 37#ifndef RSEQ_INJECT_ASM
 38#define RSEQ_INJECT_ASM(n)
 39#endif
 40
 41#ifndef RSEQ_INJECT_C
 42#define RSEQ_INJECT_C(n)
 43#endif
 44
 45#ifndef RSEQ_INJECT_INPUT
 46#define RSEQ_INJECT_INPUT
 47#endif
 48
 49#ifndef RSEQ_INJECT_CLOBBER
 50#define RSEQ_INJECT_CLOBBER
 51#endif
 52
 53#ifndef RSEQ_INJECT_FAILED
 54#define RSEQ_INJECT_FAILED
 55#endif
 56
 57#include "rseq-thread-pointer.h"
 58
 59/* Offset from the thread pointer to the rseq area. */
 60extern ptrdiff_t rseq_offset;
 61
 62/*
 63 * The rseq ABI is composed of extensible feature fields. The extensions
 64 * are done by appending additional fields at the end of the structure.
 65 * The rseq_size defines the size of the active feature set which can be
 66 * used by the application for the current rseq registration. Features
 67 * starting at offset >= rseq_size are inactive and should not be used.
 68 *
 69 * The rseq_size is the intersection between the available allocation
 70 * size for the rseq area and the feature size supported by the kernel.
 71 * unsuccessful.
 72 */
 73extern unsigned int rseq_size;
 74
 75/* Flags used during rseq registration. */
 76extern unsigned int rseq_flags;
 77
 78enum rseq_mo {
 79	RSEQ_MO_RELAXED = 0,
 80	RSEQ_MO_CONSUME = 1,	/* Unused */
 81	RSEQ_MO_ACQUIRE = 2,	/* Unused */
 82	RSEQ_MO_RELEASE = 3,
 83	RSEQ_MO_ACQ_REL = 4,	/* Unused */
 84	RSEQ_MO_SEQ_CST = 5,	/* Unused */
 85};
 86
 87enum rseq_percpu_mode {
 88	RSEQ_PERCPU_CPU_ID = 0,
 89	RSEQ_PERCPU_MM_CID = 1,
 90};
 91
 92static inline struct rseq_abi *rseq_get_abi(void)
 93{
 94	return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
 95}
 96
 97#define rseq_likely(x)		__builtin_expect(!!(x), 1)
 98#define rseq_unlikely(x)	__builtin_expect(!!(x), 0)
 99#define rseq_barrier()		__asm__ __volatile__("" : : : "memory")
100
101#define RSEQ_ACCESS_ONCE(x)	(*(__volatile__  __typeof__(x) *)&(x))
102#define RSEQ_WRITE_ONCE(x, v)	__extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
103#define RSEQ_READ_ONCE(x)	RSEQ_ACCESS_ONCE(x)
104
105#define __rseq_str_1(x)	#x
106#define __rseq_str(x)		__rseq_str_1(x)
107
108#define rseq_log(fmt, args...)						       \
109	fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
110		## args, __func__)
111
112#define rseq_bug(fmt, args...)		\
113	do {				\
114		rseq_log(fmt, ##args);	\
115		abort();		\
116	} while (0)
117
118#if defined(__x86_64__) || defined(__i386__)
119#include <rseq-x86.h>
120#elif defined(__ARMEL__)
121#include <rseq-arm.h>
122#elif defined (__AARCH64EL__)
123#include <rseq-arm64.h>
124#elif defined(__PPC__)
125#include <rseq-ppc.h>
126#elif defined(__mips__)
127#include <rseq-mips.h>
128#elif defined(__s390__)
129#include <rseq-s390.h>
130#elif defined(__riscv)
131#include <rseq-riscv.h>
132#elif defined(__or1k__)
133#include <rseq-or1k.h>
134#else
135#error unsupported target
136#endif
137
138/*
139 * Register rseq for the current thread. This needs to be called once
140 * by any thread which uses restartable sequences, before they start
141 * using restartable sequences, to ensure restartable sequences
142 * succeed. A restartable sequence executed from a non-registered
143 * thread will always fail.
144 */
145int rseq_register_current_thread(void);
146
147/*
148 * Unregister rseq for current thread.
149 */
150int rseq_unregister_current_thread(void);
151
152/*
153 * Restartable sequence fallback for reading the current CPU number.
154 */
155int32_t rseq_fallback_current_cpu(void);
156
157/*
158 * Restartable sequence fallback for reading the current node number.
159 */
160int32_t rseq_fallback_current_node(void);
161
162/*
163 * Returns true if rseq is supported.
164 */
165bool rseq_available(void);
166
167/*
168 * Values returned can be either the current CPU number, -1 (rseq is
169 * uninitialized), or -2 (rseq initialization has failed).
170 */
171static inline int32_t rseq_current_cpu_raw(void)
172{
173	return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id);
174}
175
176/*
177 * Returns a possible CPU number, which is typically the current CPU.
178 * The returned CPU number can be used to prepare for an rseq critical
179 * section, which will confirm whether the cpu number is indeed the
180 * current one, and whether rseq is initialized.
181 *
182 * The CPU number returned by rseq_cpu_start should always be validated
183 * by passing it to a rseq asm sequence, or by comparing it to the
184 * return value of rseq_current_cpu_raw() if the rseq asm sequence
185 * does not need to be invoked.
186 */
187static inline uint32_t rseq_cpu_start(void)
188{
189	return RSEQ_ACCESS_ONCE(rseq_get_abi()->cpu_id_start);
190}
191
192static inline uint32_t rseq_current_cpu(void)
193{
194	int32_t cpu;
195
196	cpu = rseq_current_cpu_raw();
197	if (rseq_unlikely(cpu < 0))
198		cpu = rseq_fallback_current_cpu();
199	return cpu;
200}
201
202static inline bool rseq_node_id_available(void)
203{
204	return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, node_id);
205}
206
207/*
208 * Current NUMA node number.
209 */
210static inline uint32_t rseq_current_node_id(void)
211{
212	assert(rseq_node_id_available());
213	return RSEQ_ACCESS_ONCE(rseq_get_abi()->node_id);
214}
215
216static inline bool rseq_mm_cid_available(void)
217{
218	return (int) rseq_size >= rseq_offsetofend(struct rseq_abi, mm_cid);
219}
220
221static inline uint32_t rseq_current_mm_cid(void)
222{
223	return RSEQ_ACCESS_ONCE(rseq_get_abi()->mm_cid);
224}
225
226static inline void rseq_clear_rseq_cs(void)
227{
228	RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0);
229}
230
231/*
232 * rseq_prepare_unload() should be invoked by each thread executing a rseq
233 * critical section at least once between their last critical section and
234 * library unload of the library defining the rseq critical section (struct
235 * rseq_cs) or the code referred to by the struct rseq_cs start_ip and
236 * post_commit_offset fields. This also applies to use of rseq in code
237 * generated by JIT: rseq_prepare_unload() should be invoked at least once by
238 * each thread executing a rseq critical section before reclaim of the memory
239 * holding the struct rseq_cs or reclaim of the code pointed to by struct
240 * rseq_cs start_ip and post_commit_offset fields.
241 */
242static inline void rseq_prepare_unload(void)
243{
244	rseq_clear_rseq_cs();
245}
246
247static inline __attribute__((always_inline))
248int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
249		       intptr_t *v, intptr_t expect,
250		       intptr_t newv, int cpu)
251{
252	if (rseq_mo != RSEQ_MO_RELAXED)
253		return -1;
254	switch (percpu_mode) {
255	case RSEQ_PERCPU_CPU_ID:
256		return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu);
257	case RSEQ_PERCPU_MM_CID:
258		return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu);
259	}
260	return -1;
261}
262
263/*
264 * Compare @v against @expectnot. When it does _not_ match, load @v
265 * into @load, and store the content of *@v + voffp into @v.
266 */
267static inline __attribute__((always_inline))
268int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
269			       intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load,
270			       int cpu)
271{
272	if (rseq_mo != RSEQ_MO_RELAXED)
273		return -1;
274	switch (percpu_mode) {
275	case RSEQ_PERCPU_CPU_ID:
276		return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu);
277	case RSEQ_PERCPU_MM_CID:
278		return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu);
279	}
280	return -1;
281}
282
283static inline __attribute__((always_inline))
284int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
285	      intptr_t *v, intptr_t count, int cpu)
286{
287	if (rseq_mo != RSEQ_MO_RELAXED)
288		return -1;
289	switch (percpu_mode) {
290	case RSEQ_PERCPU_CPU_ID:
291		return rseq_addv_relaxed_cpu_id(v, count, cpu);
292	case RSEQ_PERCPU_MM_CID:
293		return rseq_addv_relaxed_mm_cid(v, count, cpu);
294	}
295	return -1;
296}
297
298#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
299/*
300 *   pval = *(ptr+off)
301 *  *pval += inc;
302 */
303static inline __attribute__((always_inline))
304int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
305			   intptr_t *ptr, long off, intptr_t inc, int cpu)
306{
307	if (rseq_mo != RSEQ_MO_RELAXED)
308		return -1;
309	switch (percpu_mode) {
310	case RSEQ_PERCPU_CPU_ID:
311		return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu);
312	case RSEQ_PERCPU_MM_CID:
313		return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu);
314	}
315	return -1;
316}
317#endif
318
319static inline __attribute__((always_inline))
320int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
321				 intptr_t *v, intptr_t expect,
322				 intptr_t *v2, intptr_t newv2,
323				 intptr_t newv, int cpu)
324{
325	switch (rseq_mo) {
326	case RSEQ_MO_RELAXED:
327		switch (percpu_mode) {
328		case RSEQ_PERCPU_CPU_ID:
329			return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu);
330		case RSEQ_PERCPU_MM_CID:
331			return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu);
332		}
333		return -1;
334	case RSEQ_MO_RELEASE:
335		switch (percpu_mode) {
336		case RSEQ_PERCPU_CPU_ID:
337			return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu);
338		case RSEQ_PERCPU_MM_CID:
339			return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu);
340		}
341		return -1;
342	default:
343		return -1;
344	}
345}
346
347static inline __attribute__((always_inline))
348int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
349			      intptr_t *v, intptr_t expect,
350			      intptr_t *v2, intptr_t expect2,
351			      intptr_t newv, int cpu)
352{
353	if (rseq_mo != RSEQ_MO_RELAXED)
354		return -1;
355	switch (percpu_mode) {
356	case RSEQ_PERCPU_CPU_ID:
357		return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu);
358	case RSEQ_PERCPU_MM_CID:
359		return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu);
360	}
361	return -1;
362}
363
364static inline __attribute__((always_inline))
365int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
366				 intptr_t *v, intptr_t expect,
367				 void *dst, void *src, size_t len,
368				 intptr_t newv, int cpu)
369{
370	switch (rseq_mo) {
371	case RSEQ_MO_RELAXED:
372		switch (percpu_mode) {
373		case RSEQ_PERCPU_CPU_ID:
374			return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu);
375		case RSEQ_PERCPU_MM_CID:
376			return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu);
377		}
378		return -1;
379	case RSEQ_MO_RELEASE:
380		switch (percpu_mode) {
381		case RSEQ_PERCPU_CPU_ID:
382			return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu);
383		case RSEQ_PERCPU_MM_CID:
384			return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu);
385		}
386		return -1;
387	default:
388		return -1;
389	}
390}
391
392#endif  /* RSEQ_H_ */