time_bench.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Benchmarking code execution time inside the kernel
  4 *
  5 * Copyright (C) 2014, Red Hat, Inc., Jesper Dangaard Brouer
  6 *  for licensing details see kernel-base/COPYING
  7 */
  8#ifndef _LINUX_TIME_BENCH_H
  9#define _LINUX_TIME_BENCH_H
 10
 11/* Main structure used for recording a benchmark run */
 12struct time_bench_record {
 13	uint32_t version_abi;
 14	uint32_t loops;		/* Requested loop invocations */
 15	uint32_t step;		/* option for e.g. bulk invocations */
 16
 17	uint32_t flags;		/* Measurements types enabled */
 18#define TIME_BENCH_LOOP		BIT(0)
 19#define TIME_BENCH_TSC		BIT(1)
 20#define TIME_BENCH_WALLCLOCK	BIT(2)
 21#define TIME_BENCH_PMU		BIT(3)
 22
 23	uint32_t cpu; /* Used when embedded in time_bench_cpu */
 24
 25	/* Records */
 26	uint64_t invoked_cnt;	/* Returned actual invocations */
 27	uint64_t tsc_start;
 28	uint64_t tsc_stop;
 29	struct timespec64 ts_start;
 30	struct timespec64 ts_stop;
 31	/* PMU counters for instruction and cycles
 32	 * instructions counter including pipelined instructions
 33	 */
 34	uint64_t pmc_inst_start;
 35	uint64_t pmc_inst_stop;
 36	/* CPU unhalted clock counter */
 37	uint64_t pmc_clk_start;
 38	uint64_t pmc_clk_stop;
 39
 40	/* Result records */
 41	uint64_t tsc_interval;
 42	uint64_t time_start, time_stop, time_interval; /* in nanosec */
 43	uint64_t pmc_inst, pmc_clk;
 44
 45	/* Derived result records */
 46	uint64_t tsc_cycles; // +decimal?
 47	uint64_t ns_per_call_quotient, ns_per_call_decimal;
 48	uint64_t time_sec;
 49	uint32_t time_sec_remainder;
 50	uint64_t pmc_ipc_quotient, pmc_ipc_decimal; /* inst per cycle */
 51};
 52
 53/* For synchronizing parallel CPUs to run concurrently */
 54struct time_bench_sync {
 55	atomic_t nr_tests_running;
 56	struct completion start_event;
 57};
 58
 59/* Keep track of CPUs executing our bench function.
 60 *
 61 * Embed a time_bench_record for storing info per cpu
 62 */
 63struct time_bench_cpu {
 64	struct time_bench_record rec;
 65	struct time_bench_sync *sync; /* back ptr */
 66	struct task_struct *task;
 67	/* "data" opaque could have been placed in time_bench_sync,
 68	 * but to avoid any false sharing, place it per CPU
 69	 */
 70	void *data;
 71	/* Support masking outsome CPUs, mark if it ran */
 72	bool did_bench_run;
 73	/* int cpu; // note CPU stored in time_bench_record */
 74	int (*bench_func)(struct time_bench_record *record, void *data);
 75};
 76
 77/*
 78 * Below TSC assembler code is not compatible with other archs, and
 79 * can also fail on guests if cpu-flags are not correct.
 80 *
 81 * The way TSC reading is used, many iterations, does not require as
 82 * high accuracy as described below (in Intel Doc #324264).
 83 *
 84 * Considering changing to use get_cycles() (#include <asm/timex.h>).
 85 */
 86
 87/** TSC (Time-Stamp Counter) based **
 88 * Recommend reading, to understand details of reading TSC accurately:
 89 *  Intel Doc #324264, "How to Benchmark Code Execution Times on Intel"
 90 *
 91 * Consider getting exclusive ownership of CPU by using:
 92 *   unsigned long flags;
 93 *   preempt_disable();
 94 *   raw_local_irq_save(flags);
 95 *   _your_code_
 96 *   raw_local_irq_restore(flags);
 97 *   preempt_enable();
 98 *
 99 * Clobbered registers: "%rax", "%rbx", "%rcx", "%rdx"
100 *  RDTSC only change "%rax" and "%rdx" but
101 *  CPUID clears the high 32-bits of all (rax/rbx/rcx/rdx)
102 */
103static __always_inline uint64_t tsc_start_clock(void)
104{
105	/* See: Intel Doc #324264 */
106	unsigned int hi, lo;
107
108	asm volatile("CPUID\n\t"
109		     "RDTSC\n\t"
110		     "mov %%edx, %0\n\t"
111		     "mov %%eax, %1\n\t"
112		     : "=r"(hi), "=r"(lo)::"%rax", "%rbx", "%rcx", "%rdx");
113	//FIXME: on 32bit use clobbered %eax + %edx
114	return ((uint64_t)lo) | (((uint64_t)hi) << 32);
115}
116
117static __always_inline uint64_t tsc_stop_clock(void)
118{
119	/* See: Intel Doc #324264 */
120	unsigned int hi, lo;
121
122	asm volatile("RDTSCP\n\t"
123		     "mov %%edx, %0\n\t"
124		     "mov %%eax, %1\n\t"
125		     "CPUID\n\t"
126		     : "=r"(hi), "=r"(lo)::"%rax", "%rbx", "%rcx", "%rdx");
127	return ((uint64_t)lo) | (((uint64_t)hi) << 32);
128}
129
130/** Wall-clock based **
131 *
132 * use: getnstimeofday()
133 *  getnstimeofday(&rec->ts_start);
134 *  getnstimeofday(&rec->ts_stop);
135 *
136 * API changed see: Documentation/core-api/timekeeping.rst
137 *  https://www.kernel.org/doc/html/latest/core-api/timekeeping.html#c.getnstimeofday
138 *
139 * We should instead use: ktime_get_real_ts64() is a direct
140 *  replacement, but consider using monotonic time (ktime_get_ts64())
141 *  and/or a ktime_t based interface (ktime_get()/ktime_get_real()).
142 */
143
144/** PMU (Performance Monitor Unit) based **
145 *
146 * Needed for calculating: Instructions Per Cycle (IPC)
147 * - The IPC number tell how efficient the CPU pipelining were
148 */
149//lookup: perf_event_create_kernel_counter()
150
151bool time_bench_PMU_config(bool enable);
152
153/* Raw reading via rdpmc() using fixed counters
154 *
155 * From: https://github.com/andikleen/simple-pmu
156 */
157enum {
158	FIXED_SELECT = (1U << 30), /* == 0x40000000 */
159	FIXED_INST_RETIRED_ANY = 0,
160	FIXED_CPU_CLK_UNHALTED_CORE = 1,
161	FIXED_CPU_CLK_UNHALTED_REF = 2,
162};
163
164static __always_inline unsigned int long long p_rdpmc(unsigned int in)
165{
166	unsigned int d, a;
167
168	asm volatile("rdpmc" : "=d"(d), "=a"(a) : "c"(in) : "memory");
169	return ((unsigned long long)d << 32) | a;
170}
171
172/* These PMU counter needs to be enabled, but I don't have the
173 * configure code implemented.  My current hack is running:
174 *  sudo perf stat -e cycles:k -e instructions:k insmod lib/ring_queue_test.ko
175 */
176/* Reading all pipelined instruction */
177static __always_inline unsigned long long pmc_inst(void)
178{
179	return p_rdpmc(FIXED_SELECT | FIXED_INST_RETIRED_ANY);
180}
181
182/* Reading CPU clock cycles */
183static __always_inline unsigned long long pmc_clk(void)
184{
185	return p_rdpmc(FIXED_SELECT | FIXED_CPU_CLK_UNHALTED_CORE);
186}
187
188/* Raw reading via MSR rdmsr() is likely wrong
189 * FIXME: How can I know which raw MSR registers are conf for what?
190 */
191#define MSR_IA32_PCM0 0x400000C1 /* PERFCTR0 */
192#define MSR_IA32_PCM1 0x400000C2 /* PERFCTR1 */
193#define MSR_IA32_PCM2 0x400000C3
194static inline uint64_t msr_inst(unsigned long long *msr_result)
195{
196	return rdmsrq_safe(MSR_IA32_PCM0, msr_result);
197}
198
199/** Generic functions **
200 */
201bool time_bench_loop(uint32_t loops, int step, char *txt, void *data,
202		     int (*func)(struct time_bench_record *rec, void *data));
203bool time_bench_calc_stats(struct time_bench_record *rec);
204
205void time_bench_run_concurrent(uint32_t loops, int step, void *data,
206			       const struct cpumask *mask, /* Support masking outsome CPUs*/
207			       struct time_bench_sync *sync, struct time_bench_cpu *cpu_tasks,
208			       int (*func)(struct time_bench_record *record, void *data));
209void time_bench_print_stats_cpumask(const char *desc,
210				    struct time_bench_cpu *cpu_tasks,
211				    const struct cpumask *mask);
212
213//FIXME: use rec->flags to select measurement, should be MACRO
214static __always_inline void time_bench_start(struct time_bench_record *rec)
215{
216	//getnstimeofday(&rec->ts_start);
217	ktime_get_real_ts64(&rec->ts_start);
218	if (rec->flags & TIME_BENCH_PMU) {
219		rec->pmc_inst_start = pmc_inst();
220		rec->pmc_clk_start = pmc_clk();
221	}
222	rec->tsc_start = tsc_start_clock();
223}
224
225static __always_inline void time_bench_stop(struct time_bench_record *rec,
226					    uint64_t invoked_cnt)
227{
228	rec->tsc_stop = tsc_stop_clock();
229	if (rec->flags & TIME_BENCH_PMU) {
230		rec->pmc_inst_stop = pmc_inst();
231		rec->pmc_clk_stop = pmc_clk();
232	}
233	//getnstimeofday(&rec->ts_stop);
234	ktime_get_real_ts64(&rec->ts_stop);
235	rec->invoked_cnt = invoked_cnt;
236}
237
238#endif /* _LINUX_TIME_BENCH_H */