vclock_gettime.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright 2006 Andi Kleen, SUSE Labs.
  4 *
  5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
  6 *
  7 * The code should have no internal unresolved relocations.
  8 * Check with readelf after changing.
  9 * Also alternative() doesn't work.
 10 */
 11/*
 12 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
 13 */
 14
 15#include <linux/kernel.h>
 16#include <linux/time.h>
 17#include <linux/string.h>
 18#include <asm/io.h>
 19#include <asm/unistd.h>
 20#include <asm/timex.h>
 21#include <asm/clocksource.h>
 22#include <asm/vvar.h>
 23
 24#ifdef	CONFIG_SPARC64
 25#define SYSCALL_STRING							\
 26	"ta	0x6d;"							\
 27	"bcs,a	1f;"							\
 28	" sub	%%g0, %%o0, %%o0;"					\
 29	"1:"
 30#else
 31#define SYSCALL_STRING							\
 32	"ta	0x10;"							\
 33	"bcs,a	1f;"							\
 34	" sub	%%g0, %%o0, %%o0;"					\
 35	"1:"
 36#endif
 37
 38#define SYSCALL_CLOBBERS						\
 39	"f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",			\
 40	"f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",		\
 41	"f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",		\
 42	"f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",		\
 43	"f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",		\
 44	"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",		\
 45	"cc", "memory"
 46
 47/*
 48 * Compute the vvar page's address in the process address space, and return it
 49 * as a pointer to the vvar_data.
 50 */
 51notrace static __always_inline struct vvar_data *get_vvar_data(void)
 52{
 53	unsigned long ret;
 54
 55	/*
 56	 * vdso data page is the first vDSO page so grab the PC
 57	 * and move up a page to get to the data page.
 58	 */
 59	__asm__("rd %%pc, %0" : "=r" (ret));
 60	ret &= ~(8192 - 1);
 61	ret -= 8192;
 62
 63	return (struct vvar_data *) ret;
 64}
 65
 66notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
 67{
 68	register long num __asm__("g1") = __NR_clock_gettime;
 69	register long o0 __asm__("o0") = clock;
 70	register long o1 __asm__("o1") = (long) ts;
 71
 72	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
 73			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
 74	return o0;
 75}
 76
 77notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 78{
 79	register long num __asm__("g1") = __NR_gettimeofday;
 80	register long o0 __asm__("o0") = (long) tv;
 81	register long o1 __asm__("o1") = (long) tz;
 82
 83	__asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
 84			     "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
 85	return o0;
 86}
 87
 88#ifdef	CONFIG_SPARC64
 89notrace static __always_inline u64 __shr64(u64 val, int amt)
 90{
 91	return val >> amt;
 92}
 93
 94notrace static __always_inline u64 vread_tick(void)
 95{
 96	u64	ret;
 97
 98	__asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
 99	return ret;
100}
101
102notrace static __always_inline u64 vread_tick_stick(void)
103{
104	u64	ret;
105
106	__asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
107	return ret;
108}
109#else
110notrace static __always_inline u64 __shr64(u64 val, int amt)
111{
112	u64 ret;
113
114	__asm__ __volatile__("sllx %H1, 32, %%g1\n\t"
115			     "srl %L1, 0, %L1\n\t"
116			     "or %%g1, %L1, %%g1\n\t"
117			     "srlx %%g1, %2, %L0\n\t"
118			     "srlx %L0, 32, %H0"
119			     : "=r" (ret)
120			     : "r" (val), "r" (amt)
121			     : "g1");
122	return ret;
123}
124
125notrace static __always_inline u64 vread_tick(void)
126{
127	register unsigned long long ret asm("o4");
128
129	__asm__ __volatile__("rd %%tick, %L0\n\t"
130			     "srlx %L0, 32, %H0"
131			     : "=r" (ret));
132	return ret;
133}
134
135notrace static __always_inline u64 vread_tick_stick(void)
136{
137	register unsigned long long ret asm("o4");
138
139	__asm__ __volatile__("rd %%asr24, %L0\n\t"
140			     "srlx %L0, 32, %H0"
141			     : "=r" (ret));
142	return ret;
143}
144#endif
145
146notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
147{
148	u64 v;
149	u64 cycles;
150
151	cycles = vread_tick();
152	v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
153	return v * vvar->clock.mult;
154}
155
156notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
157{
158	u64 v;
159	u64 cycles;
160
161	cycles = vread_tick_stick();
162	v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
163	return v * vvar->clock.mult;
164}
165
166notrace static __always_inline int do_realtime(struct vvar_data *vvar,
167					       struct __kernel_old_timespec *ts)
168{
169	unsigned long seq;
170	u64 ns;
171
172	do {
173		seq = vvar_read_begin(vvar);
174		ts->tv_sec = vvar->wall_time_sec;
175		ns = vvar->wall_time_snsec;
176		ns += vgetsns(vvar);
177		ns = __shr64(ns, vvar->clock.shift);
178	} while (unlikely(vvar_read_retry(vvar, seq)));
179
180	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
181	ts->tv_nsec = ns;
182
183	return 0;
184}
185
186notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
187						     struct __kernel_old_timespec *ts)
188{
189	unsigned long seq;
190	u64 ns;
191
192	do {
193		seq = vvar_read_begin(vvar);
194		ts->tv_sec = vvar->wall_time_sec;
195		ns = vvar->wall_time_snsec;
196		ns += vgetsns_stick(vvar);
197		ns = __shr64(ns, vvar->clock.shift);
198	} while (unlikely(vvar_read_retry(vvar, seq)));
199
200	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
201	ts->tv_nsec = ns;
202
203	return 0;
204}
205
206notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
207						struct __kernel_old_timespec *ts)
208{
209	unsigned long seq;
210	u64 ns;
211
212	do {
213		seq = vvar_read_begin(vvar);
214		ts->tv_sec = vvar->monotonic_time_sec;
215		ns = vvar->monotonic_time_snsec;
216		ns += vgetsns(vvar);
217		ns = __shr64(ns, vvar->clock.shift);
218	} while (unlikely(vvar_read_retry(vvar, seq)));
219
220	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
221	ts->tv_nsec = ns;
222
223	return 0;
224}
225
226notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
227						      struct __kernel_old_timespec *ts)
228{
229	unsigned long seq;
230	u64 ns;
231
232	do {
233		seq = vvar_read_begin(vvar);
234		ts->tv_sec = vvar->monotonic_time_sec;
235		ns = vvar->monotonic_time_snsec;
236		ns += vgetsns_stick(vvar);
237		ns = __shr64(ns, vvar->clock.shift);
238	} while (unlikely(vvar_read_retry(vvar, seq)));
239
240	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
241	ts->tv_nsec = ns;
242
243	return 0;
244}
245
246notrace static int do_realtime_coarse(struct vvar_data *vvar,
247				      struct __kernel_old_timespec *ts)
248{
249	unsigned long seq;
250
251	do {
252		seq = vvar_read_begin(vvar);
253		ts->tv_sec = vvar->wall_time_coarse_sec;
254		ts->tv_nsec = vvar->wall_time_coarse_nsec;
255	} while (unlikely(vvar_read_retry(vvar, seq)));
256	return 0;
257}
258
259notrace static int do_monotonic_coarse(struct vvar_data *vvar,
260				       struct __kernel_old_timespec *ts)
261{
262	unsigned long seq;
263
264	do {
265		seq = vvar_read_begin(vvar);
266		ts->tv_sec = vvar->monotonic_time_coarse_sec;
267		ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
268	} while (unlikely(vvar_read_retry(vvar, seq)));
269
270	return 0;
271}
272
273notrace int
274__vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
275{
276	struct vvar_data *vvd = get_vvar_data();
277
278	switch (clock) {
279	case CLOCK_REALTIME:
280		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
281			break;
282		return do_realtime(vvd, ts);
283	case CLOCK_MONOTONIC:
284		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
285			break;
286		return do_monotonic(vvd, ts);
287	case CLOCK_REALTIME_COARSE:
288		return do_realtime_coarse(vvd, ts);
289	case CLOCK_MONOTONIC_COARSE:
290		return do_monotonic_coarse(vvd, ts);
291	}
292	/*
293	 * Unknown clock ID ? Fall back to the syscall.
294	 */
295	return vdso_fallback_gettime(clock, ts);
296}
297int
298clock_gettime(clockid_t, struct __kernel_old_timespec *)
299	__attribute__((weak, alias("__vdso_clock_gettime")));
300
301notrace int
302__vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
303{
304	struct vvar_data *vvd = get_vvar_data();
305
306	switch (clock) {
307	case CLOCK_REALTIME:
308		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
309			break;
310		return do_realtime_stick(vvd, ts);
311	case CLOCK_MONOTONIC:
312		if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
313			break;
314		return do_monotonic_stick(vvd, ts);
315	case CLOCK_REALTIME_COARSE:
316		return do_realtime_coarse(vvd, ts);
317	case CLOCK_MONOTONIC_COARSE:
318		return do_monotonic_coarse(vvd, ts);
319	}
320	/*
321	 * Unknown clock ID ? Fall back to the syscall.
322	 */
323	return vdso_fallback_gettime(clock, ts);
324}
325
326notrace int
327__vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
328{
329	struct vvar_data *vvd = get_vvar_data();
330
331	if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
332		if (likely(tv != NULL)) {
333			union tstv_t {
334				struct __kernel_old_timespec ts;
335				struct __kernel_old_timeval tv;
336			} *tstv = (union tstv_t *) tv;
337			do_realtime(vvd, &tstv->ts);
338			/*
339			 * Assign before dividing to ensure that the division is
340			 * done in the type of tv_usec, not tv_nsec.
341			 *
342			 * There cannot be > 1 billion usec in a second:
343			 * do_realtime() has already distributed such overflow
344			 * into tv_sec.  So we can assign it to an int safely.
345			 */
346			tstv->tv.tv_usec = tstv->ts.tv_nsec;
347			tstv->tv.tv_usec /= 1000;
348		}
349		if (unlikely(tz != NULL)) {
350			/* Avoid memcpy. Some old compilers fail to inline it */
351			tz->tz_minuteswest = vvd->tz_minuteswest;
352			tz->tz_dsttime = vvd->tz_dsttime;
353		}
354		return 0;
355	}
356	return vdso_fallback_gettimeofday(tv, tz);
357}
358int
359gettimeofday(struct __kernel_old_timeval *, struct timezone *)
360	__attribute__((weak, alias("__vdso_gettimeofday")));
361
362notrace int
363__vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
364{
365	struct vvar_data *vvd = get_vvar_data();
366
367	if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
368		if (likely(tv != NULL)) {
369			union tstv_t {
370				struct __kernel_old_timespec ts;
371				struct __kernel_old_timeval tv;
372			} *tstv = (union tstv_t *) tv;
373			do_realtime_stick(vvd, &tstv->ts);
374			/*
375			 * Assign before dividing to ensure that the division is
376			 * done in the type of tv_usec, not tv_nsec.
377			 *
378			 * There cannot be > 1 billion usec in a second:
379			 * do_realtime() has already distributed such overflow
380			 * into tv_sec.  So we can assign it to an int safely.
381			 */
382			tstv->tv.tv_usec = tstv->ts.tv_nsec;
383			tstv->tv.tv_usec /= 1000;
384		}
385		if (unlikely(tz != NULL)) {
386			/* Avoid memcpy. Some old compilers fail to inline it */
387			tz->tz_minuteswest = vvd->tz_minuteswest;
388			tz->tz_dsttime = vvd->tz_dsttime;
389		}
390		return 0;
391	}
392	return vdso_fallback_gettimeofday(tv, tz);
393}