tsc.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/*
  3 * x86 TSC related functions
  4 */
  5#ifndef _ASM_X86_TSC_H
  6#define _ASM_X86_TSC_H
  7
  8#include <asm/asm.h>
  9#include <asm/cpufeature.h>
 10#include <asm/processor.h>
 11#include <asm/msr.h>
 12
 13/**
 14 * rdtsc() - returns the current TSC without ordering constraints
 15 *
 16 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
 17 * only ordering constraint it supplies is the ordering implied by
 18 * "asm volatile": it will put the RDTSC in the place you expect.  The
 19 * CPU can and will speculatively execute that RDTSC, though, so the
 20 * results can be non-monotonic if compared on different CPUs.
 21 */
 22static __always_inline u64 rdtsc(void)
 23{
 24	EAX_EDX_DECLARE_ARGS(val, low, high);
 25
 26	asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
 27
 28	return EAX_EDX_VAL(val, low, high);
 29}
 30
 31/**
 32 * rdtsc_ordered() - read the current TSC in program order
 33 *
 34 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
 35 * It is ordered like a load to a global in-memory counter.  It should
 36 * be impossible to observe non-monotonic rdtsc_unordered() behavior
 37 * across multiple CPUs as long as the TSC is synced.
 38 */
 39static __always_inline u64 rdtsc_ordered(void)
 40{
 41	EAX_EDX_DECLARE_ARGS(val, low, high);
 42
 43	/*
 44	 * The RDTSC instruction is not ordered relative to memory
 45	 * access.  The Intel SDM and the AMD APM are both vague on this
 46	 * point, but empirically an RDTSC instruction can be
 47	 * speculatively executed before prior loads.  An RDTSC
 48	 * immediately after an appropriate barrier appears to be
 49	 * ordered as a normal load, that is, it provides the same
 50	 * ordering guarantees as reading from a global memory location
 51	 * that some other imaginary CPU is updating continuously with a
 52	 * time stamp.
 53	 *
 54	 * Thus, use the preferred barrier on the respective CPU, aiming for
 55	 * RDTSCP as the default.
 56	 */
 57	asm volatile(ALTERNATIVE_2("rdtsc",
 58				   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
 59				   "rdtscp", X86_FEATURE_RDTSCP)
 60			: EAX_EDX_RET(val, low, high)
 61			/* RDTSCP clobbers ECX with MSR_TSC_AUX. */
 62			:: "ecx");
 63
 64	return EAX_EDX_VAL(val, low, high);
 65}
 66
 67/*
 68 * Standard way to access the cycle counter.
 69 */
 70typedef unsigned long long cycles_t;
 71
 72extern unsigned int cpu_khz;
 73extern unsigned int tsc_khz;
 74
 75extern void disable_TSC(void);
 76
 77static inline cycles_t get_cycles(void)
 78{
 79	if (!IS_ENABLED(CONFIG_X86_TSC) &&
 80	    !cpu_feature_enabled(X86_FEATURE_TSC))
 81		return 0;
 82	return rdtsc();
 83}
 84#define get_cycles get_cycles
 85
 86extern void tsc_early_init(void);
 87extern void tsc_init(void);
 88extern void mark_tsc_unstable(char *reason);
 89extern int unsynchronized_tsc(void);
 90extern int check_tsc_unstable(void);
 91extern void mark_tsc_async_resets(char *reason);
 92extern unsigned long native_calibrate_cpu_early(void);
 93extern unsigned long native_calibrate_tsc(void);
 94extern unsigned long long native_sched_clock_from_tsc(u64 tsc);
 95
 96extern int tsc_clocksource_reliable;
 97#ifdef CONFIG_X86_TSC
 98extern bool tsc_async_resets;
 99#else
100# define tsc_async_resets	false
101#endif
102
103/*
104 * Boot-time check whether the TSCs are synchronized across
105 * all CPUs/cores:
106 */
107#ifdef CONFIG_X86_TSC
108extern bool tsc_store_and_check_tsc_adjust(bool bootcpu);
109extern void tsc_verify_tsc_adjust(bool resume);
110extern void check_tsc_sync_target(void);
111#else
112static inline bool tsc_store_and_check_tsc_adjust(bool bootcpu) { return false; }
113static inline void tsc_verify_tsc_adjust(bool resume) { }
114static inline void check_tsc_sync_target(void) { }
115#endif
116
117extern int notsc_setup(char *);
118extern void tsc_save_sched_clock_state(void);
119extern void tsc_restore_sched_clock_state(void);
120
121unsigned long cpu_khz_from_msr(void);
122
123#endif /* _ASM_X86_TSC_H */