calibrate.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* calibrate.c: default delay calibration
  3 *
  4 * Excised from init/main.c
  5 *  Copyright (C) 1991, 1992  Linus Torvalds
  6 */
  7
  8#include <linux/delay.h>
  9#include <linux/init.h>
 10#include <linux/jiffies.h>
 11#include <linux/kstrtox.h>
 12#include <linux/percpu.h>
 13#include <linux/printk.h>
 14#include <linux/smp.h>
 15#include <linux/stddef.h>
 16#include <linux/timex.h>
 17
 18unsigned long lpj_fine;
 19unsigned long preset_lpj;
 20
 21static int __init lpj_setup(char *str)
 22{
 23	return kstrtoul(str, 0, &preset_lpj) == 0;
 24}
 25
 26__setup("lpj=", lpj_setup);
 27
 28#ifdef ARCH_HAS_READ_CURRENT_TIMER
 29
 30/* This routine uses the read_current_timer() routine and gets the
 31 * loops per jiffy directly, instead of guessing it using delay().
 32 * Also, this code tries to handle non-maskable asynchronous events
 33 * (like SMIs)
 34 */
 35#define DELAY_CALIBRATION_TICKS			((HZ < 100) ? 1 : (HZ/100))
 36#define MAX_DIRECT_CALIBRATION_RETRIES		5
 37
 38static unsigned long calibrate_delay_direct(void)
 39{
 40	unsigned long pre_start, start, post_start;
 41	unsigned long pre_end, end, post_end;
 42	unsigned long start_jiffies;
 43	unsigned long timer_rate_min, timer_rate_max;
 44	unsigned long good_timer_sum = 0;
 45	unsigned long good_timer_count = 0;
 46	unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES];
 47	int max = -1; /* index of measured_times with max/min values or not set */
 48	int min = -1;
 49	int i;
 50
 51	if (read_current_timer(&pre_start) < 0 )
 52		return 0;
 53
 54	/*
 55	 * A simple loop like
 56	 *	while ( jiffies < start_jiffies+1)
 57	 *		start = read_current_timer();
 58	 * will not do. As we don't really know whether jiffy switch
 59	 * happened first or timer_value was read first. And some asynchronous
 60	 * event can happen between these two events introducing errors in lpj.
 61	 *
 62	 * So, we do
 63	 * 1. pre_start <- When we are sure that jiffy switch hasn't happened
 64	 * 2. check jiffy switch
 65	 * 3. start <- timer value before or after jiffy switch
 66	 * 4. post_start <- When we are sure that jiffy switch has happened
 67	 *
 68	 * Note, we don't know anything about order of 2 and 3.
 69	 * Now, by looking at post_start and pre_start difference, we can
 70	 * check whether any asynchronous event happened or not
 71	 */
 72
 73	for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
 74		pre_start = 0;
 75		read_current_timer(&start);
 76		start_jiffies = jiffies;
 77		while (time_before_eq(jiffies, start_jiffies + 1)) {
 78			pre_start = start;
 79			read_current_timer(&start);
 80		}
 81		read_current_timer(&post_start);
 82
 83		pre_end = 0;
 84		end = post_start;
 85		while (time_before_eq(jiffies, start_jiffies + 1 +
 86					       DELAY_CALIBRATION_TICKS)) {
 87			pre_end = end;
 88			read_current_timer(&end);
 89		}
 90		read_current_timer(&post_end);
 91
 92		timer_rate_max = (post_end - pre_start) /
 93					DELAY_CALIBRATION_TICKS;
 94		timer_rate_min = (pre_end - post_start) /
 95					DELAY_CALIBRATION_TICKS;
 96
 97		/*
 98		 * If the upper limit and lower limit of the timer_rate is
 99		 * >= 12.5% apart, redo calibration.
100		 */
101		if (start >= post_end)
102			printk(KERN_NOTICE "calibrate_delay_direct() ignoring "
103					"timer_rate as we had a TSC wrap around"
104					" start=%lu >=post_end=%lu\n",
105				start, post_end);
106		if (start < post_end && pre_start != 0 && pre_end != 0 &&
107		    (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) {
108			good_timer_count++;
109			good_timer_sum += timer_rate_max;
110			measured_times[i] = timer_rate_max;
111			if (max < 0 || timer_rate_max > measured_times[max])
112				max = i;
113			if (min < 0 || timer_rate_max < measured_times[min])
114				min = i;
115		} else
116			measured_times[i] = 0;
117
118	}
119
120	/*
121	 * Find the maximum & minimum - if they differ too much throw out the
122	 * one with the largest difference from the mean and try again...
123	 */
124	while (good_timer_count > 1) {
125		unsigned long estimate;
126		unsigned long maxdiff;
127
128		/* compute the estimate */
129		estimate = (good_timer_sum/good_timer_count);
130		maxdiff = estimate >> 3;
131
132		/* if range is within 12% let's take it */
133		if ((measured_times[max] - measured_times[min]) < maxdiff)
134			return estimate;
135
136		/* ok - drop the worse value and try again... */
137		good_timer_sum = 0;
138		good_timer_count = 0;
139		if ((measured_times[max] - estimate) <
140				(estimate - measured_times[min])) {
141			printk(KERN_NOTICE "calibrate_delay_direct() dropping "
142					"min bogoMips estimate %d = %lu\n",
143				min, measured_times[min]);
144			measured_times[min] = 0;
145			min = max;
146		} else {
147			printk(KERN_NOTICE "calibrate_delay_direct() dropping "
148					"max bogoMips estimate %d = %lu\n",
149				max, measured_times[max]);
150			measured_times[max] = 0;
151			max = min;
152		}
153
154		for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
155			if (measured_times[i] == 0)
156				continue;
157			good_timer_count++;
158			good_timer_sum += measured_times[i];
159			if (measured_times[i] < measured_times[min])
160				min = i;
161			if (measured_times[i] > measured_times[max])
162				max = i;
163		}
164
165	}
166
167	printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good "
168	       "estimate for loops_per_jiffy.\nProbably due to long platform "
169		"interrupts. Consider using \"lpj=\" boot option.\n");
170	return 0;
171}
172#else
173static unsigned long calibrate_delay_direct(void)
174{
175	return 0;
176}
177#endif
178
179/*
180 * This is the number of bits of precision for the loops_per_jiffy.  Each
181 * time we refine our estimate after the first takes 1.5/HZ seconds, so try
182 * to start with a good estimate.
183 * For the boot cpu we can skip the delay calibration and assign it a value
184 * calculated based on the timer frequency.
185 * For the rest of the CPUs we cannot assume that the timer frequency is same as
186 * the cpu frequency, hence do the calibration for those.
187 */
188#define LPS_PREC 8
189
190static unsigned long calibrate_delay_converge(void)
191{
192	/* First stage - slowly accelerate to find initial bounds */
193	unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit;
194	int trials = 0, band = 0, trial_in_band = 0;
195
196	lpj = (1<<12);
197
198	/* wait for "start of" clock tick */
199	ticks = jiffies;
200	while (ticks == jiffies)
201		; /* nothing */
202	/* Go .. */
203	ticks = jiffies;
204	do {
205		if (++trial_in_band == (1<<band)) {
206			++band;
207			trial_in_band = 0;
208		}
209		__delay(lpj * band);
210		trials += band;
211	} while (ticks == jiffies);
212	/*
213	 * We overshot, so retreat to a clear underestimate. Then estimate
214	 * the largest likely undershoot. This defines our chop bounds.
215	 */
216	trials -= band;
217	loopadd_base = lpj * band;
218	lpj_base = lpj * trials;
219
220recalibrate:
221	lpj = lpj_base;
222	loopadd = loopadd_base;
223
224	/*
225	 * Do a binary approximation to get lpj set to
226	 * equal one clock (up to LPS_PREC bits)
227	 */
228	chop_limit = lpj >> LPS_PREC;
229	while (loopadd > chop_limit) {
230		lpj += loopadd;
231		ticks = jiffies;
232		while (ticks == jiffies)
233			; /* nothing */
234		ticks = jiffies;
235		__delay(lpj);
236		if (jiffies != ticks)	/* longer than 1 tick */
237			lpj -= loopadd;
238		loopadd >>= 1;
239	}
240	/*
241	 * If we incremented every single time possible, presume we've
242	 * massively underestimated initially, and retry with a higher
243	 * start, and larger range. (Only seen on x86_64, due to SMIs)
244	 */
245	if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) {
246		lpj_base = lpj;
247		loopadd_base <<= 2;
248		goto recalibrate;
249	}
250
251	return lpj;
252}
253
254static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 };
255
256/*
257 * Check if cpu calibration delay is already known. For example,
258 * some processors with multi-core sockets may have all cores
259 * with the same calibration delay.
260 *
261 * Architectures should override this function if a faster calibration
262 * method is available.
263 */
264unsigned long __attribute__((weak)) calibrate_delay_is_known(void)
265{
266	return 0;
267}
268
269/*
270 * Indicate the cpu delay calibration is done. This can be used by
271 * architectures to stop accepting delay timer registrations after this point.
272 */
273
274void __attribute__((weak)) calibration_delay_done(void)
275{
276}
277
278void calibrate_delay(void)
279{
280	unsigned long lpj;
281	static bool printed;
282	int this_cpu = smp_processor_id();
283
284	if (per_cpu(cpu_loops_per_jiffy, this_cpu)) {
285		lpj = per_cpu(cpu_loops_per_jiffy, this_cpu);
286		if (!printed)
287			pr_info("Calibrating delay loop (skipped) "
288				"already calibrated this CPU");
289	} else if (preset_lpj) {
290		lpj = preset_lpj;
291		if (!printed)
292			pr_info("Calibrating delay loop (skipped) "
293				"preset value.. ");
294	} else if ((!printed) && lpj_fine) {
295		lpj = lpj_fine;
296		pr_info("Calibrating delay loop (skipped), "
297			"value calculated using timer frequency.. ");
298	} else if ((lpj = calibrate_delay_is_known())) {
299		;
300	} else if ((lpj = calibrate_delay_direct()) != 0) {
301		if (!printed)
302			pr_info("Calibrating delay using timer "
303				"specific routine.. ");
304	} else {
305		if (!printed)
306			pr_info("Calibrating delay loop... ");
307		lpj = calibrate_delay_converge();
308	}
309	per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj;
310	if (!printed)
311		pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n",
312			lpj/(500000/HZ),
313			(lpj/(5000/HZ)) % 100, lpj);
314
315	loops_per_jiffy = lpj;
316	printed = true;
317
318	calibration_delay_done();
319}