timekeeping.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 *  Kernel timekeeping code and accessor functions. Based on code from
   4 *  timer.c, moved in commit 8524070b7982.
   5 */
   6#include <linux/timekeeper_internal.h>
   7#include <linux/module.h>
   8#include <linux/interrupt.h>
   9#include <linux/kobject.h>
  10#include <linux/percpu.h>
  11#include <linux/init.h>
  12#include <linux/mm.h>
  13#include <linux/nmi.h>
  14#include <linux/sched.h>
  15#include <linux/sched/loadavg.h>
  16#include <linux/sched/clock.h>
  17#include <linux/syscore_ops.h>
  18#include <linux/clocksource.h>
  19#include <linux/jiffies.h>
  20#include <linux/time.h>
  21#include <linux/timex.h>
  22#include <linux/tick.h>
  23#include <linux/stop_machine.h>
  24#include <linux/pvclock_gtod.h>
  25#include <linux/compiler.h>
  26#include <linux/audit.h>
  27#include <linux/random.h>
  28
  29#include <vdso/auxclock.h>
  30
  31#include "tick-internal.h"
  32#include "ntp_internal.h"
  33#include "timekeeping_internal.h"
  34
  35#define TK_CLEAR_NTP		(1 << 0)
  36#define TK_CLOCK_WAS_SET	(1 << 1)
  37
  38#define TK_UPDATE_ALL		(TK_CLEAR_NTP | TK_CLOCK_WAS_SET)
  39
  40enum timekeeping_adv_mode {
  41	/* Update timekeeper when a tick has passed */
  42	TK_ADV_TICK,
  43
  44	/* Update timekeeper on a direct frequency change */
  45	TK_ADV_FREQ
  46};
  47
  48/*
  49 * The most important data for readout fits into a single 64 byte
  50 * cache line.
  51 */
  52struct tk_data {
  53	seqcount_raw_spinlock_t	seq;
  54	struct timekeeper	timekeeper;
  55	struct timekeeper	shadow_timekeeper;
  56	raw_spinlock_t		lock;
  57} ____cacheline_aligned;
  58
  59static struct tk_data timekeeper_data[TIMEKEEPERS_MAX];
  60
  61/* The core timekeeper */
  62#define tk_core		(timekeeper_data[TIMEKEEPER_CORE])
  63
  64#ifdef CONFIG_POSIX_AUX_CLOCKS
  65static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
  66{
  67	return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts);
  68}
  69
  70static inline bool tk_is_aux(const struct timekeeper *tk)
  71{
  72	return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST;
  73}
  74#else
  75static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts)
  76{
  77	return false;
  78}
  79
  80static inline bool tk_is_aux(const struct timekeeper *tk)
  81{
  82	return false;
  83}
  84#endif
  85
  86static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs)
  87{
  88	tk->offs_aux = offs;
  89	tk->monotonic_to_aux = ktime_to_timespec64(offs);
  90}
  91
  92/* flag for if timekeeping is suspended */
  93int __read_mostly timekeeping_suspended;
  94
  95/**
  96 * struct tk_fast - NMI safe timekeeper
  97 * @seq:	Sequence counter for protecting updates. The lowest bit
  98 *		is the index for the tk_read_base array
  99 * @base:	tk_read_base array. Access is indexed by the lowest bit of
 100 *		@seq.
 101 *
 102 * See @update_fast_timekeeper() below.
 103 */
 104struct tk_fast {
 105	seqcount_latch_t	seq;
 106	struct tk_read_base	base[2];
 107};
 108
 109/* Suspend-time cycles value for halted fast timekeeper. */
 110static u64 cycles_at_suspend;
 111
 112static u64 dummy_clock_read(struct clocksource *cs)
 113{
 114	if (timekeeping_suspended)
 115		return cycles_at_suspend;
 116	return local_clock();
 117}
 118
 119static struct clocksource dummy_clock = {
 120	.read = dummy_clock_read,
 121};
 122
 123/*
 124 * Boot time initialization which allows local_clock() to be utilized
 125 * during early boot when clocksources are not available. local_clock()
 126 * returns nanoseconds already so no conversion is required, hence mult=1
 127 * and shift=0. When the first proper clocksource is installed then
 128 * the fast time keepers are updated with the correct values.
 129 */
 130#define FAST_TK_INIT						\
 131	{							\
 132		.clock		= &dummy_clock,			\
 133		.mask		= CLOCKSOURCE_MASK(64),		\
 134		.mult		= 1,				\
 135		.shift		= 0,				\
 136	}
 137
 138static struct tk_fast tk_fast_mono ____cacheline_aligned = {
 139	.seq     = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
 140	.base[0] = FAST_TK_INIT,
 141	.base[1] = FAST_TK_INIT,
 142};
 143
 144static struct tk_fast tk_fast_raw  ____cacheline_aligned = {
 145	.seq     = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
 146	.base[0] = FAST_TK_INIT,
 147	.base[1] = FAST_TK_INIT,
 148};
 149
 150#ifdef CONFIG_POSIX_AUX_CLOCKS
 151static __init void tk_aux_setup(void);
 152static void tk_aux_update_clocksource(void);
 153static void tk_aux_advance(void);
 154#else
 155static inline void tk_aux_setup(void) { }
 156static inline void tk_aux_update_clocksource(void) { }
 157static inline void tk_aux_advance(void) { }
 158#endif
 159
 160unsigned long timekeeper_lock_irqsave(void)
 161{
 162	unsigned long flags;
 163
 164	raw_spin_lock_irqsave(&tk_core.lock, flags);
 165	return flags;
 166}
 167
 168void timekeeper_unlock_irqrestore(unsigned long flags)
 169{
 170	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
 171}
 172
 173/*
 174 * Multigrain timestamps require tracking the latest fine-grained timestamp
 175 * that has been issued, and never returning a coarse-grained timestamp that is
 176 * earlier than that value.
 177 *
 178 * mg_floor represents the latest fine-grained time that has been handed out as
 179 * a file timestamp on the system. This is tracked as a monotonic ktime_t, and
 180 * converted to a realtime clock value on an as-needed basis.
 181 *
 182 * Maintaining mg_floor ensures the multigrain interfaces never issue a
 183 * timestamp earlier than one that has been previously issued.
 184 *
 185 * The exception to this rule is when there is a backward realtime clock jump. If
 186 * such an event occurs, a timestamp can appear to be earlier than a previous one.
 187 */
 188static __cacheline_aligned_in_smp atomic64_t mg_floor;
 189
 190static inline void tk_normalize_xtime(struct timekeeper *tk)
 191{
 192	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
 193		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
 194		tk->xtime_sec++;
 195	}
 196	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
 197		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
 198		tk->raw_sec++;
 199	}
 200}
 201
 202static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
 203{
 204	struct timespec64 ts;
 205
 206	ts.tv_sec = tk->xtime_sec;
 207	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
 208	return ts;
 209}
 210
 211static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk)
 212{
 213	struct timespec64 ts;
 214
 215	ts.tv_sec = tk->xtime_sec;
 216	ts.tv_nsec = tk->coarse_nsec;
 217	return ts;
 218}
 219
 220/*
 221 * Update the nanoseconds part for the coarse time keepers. They can't rely
 222 * on xtime_nsec because xtime_nsec could be adjusted by a small negative
 223 * amount when the multiplication factor of the clock is adjusted, which
 224 * could cause the coarse clocks to go slightly backwards. See
 225 * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse
 226 * clockids which only is updated when the clock has been set or  we have
 227 * accumulated time.
 228 */
 229static inline void tk_update_coarse_nsecs(struct timekeeper *tk)
 230{
 231	tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
 232}
 233
 234static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
 235{
 236	tk->xtime_sec = ts->tv_sec;
 237	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
 238	tk_update_coarse_nsecs(tk);
 239}
 240
 241static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
 242{
 243	tk->xtime_sec += ts->tv_sec;
 244	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
 245	tk_normalize_xtime(tk);
 246	tk_update_coarse_nsecs(tk);
 247}
 248
 249static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
 250{
 251	struct timespec64 tmp;
 252
 253	/*
 254	 * Verify consistency of: offset_real = -wall_to_monotonic
 255	 * before modifying anything
 256	 */
 257	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
 258					-tk->wall_to_monotonic.tv_nsec);
 259	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
 260	tk->wall_to_monotonic = wtm;
 261	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
 262	/* Paired with READ_ONCE() in ktime_mono_to_any() */
 263	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));
 264	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));
 265}
 266
 267static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
 268{
 269	/* Paired with READ_ONCE() in ktime_mono_to_any() */
 270	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));
 271	/*
 272	 * Timespec representation for VDSO update to avoid 64bit division
 273	 * on every update.
 274	 */
 275	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
 276}
 277
 278/*
 279 * tk_clock_read - atomic clocksource read() helper
 280 *
 281 * This helper is necessary to use in the read paths because, while the
 282 * seqcount ensures we don't return a bad value while structures are updated,
 283 * it doesn't protect from potential crashes. There is the possibility that
 284 * the tkr's clocksource may change between the read reference, and the
 285 * clock reference passed to the read function.  This can cause crashes if
 286 * the wrong clocksource is passed to the wrong read function.
 287 * This isn't necessary to use when holding the tk_core.lock or doing
 288 * a read of the fast-timekeeper tkrs (which is protected by its own locking
 289 * and update logic).
 290 */
 291static inline u64 tk_clock_read(const struct tk_read_base *tkr)
 292{
 293	struct clocksource *clock = READ_ONCE(tkr->clock);
 294
 295	return clock->read(clock);
 296}
 297
 298/**
 299 * tk_setup_internals - Set up internals to use clocksource clock.
 300 *
 301 * @tk:		The target timekeeper to setup.
 302 * @clock:		Pointer to clocksource.
 303 *
 304 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
 305 * pair and interval request.
 306 *
 307 * Unless you're the timekeeping code, you should not be using this!
 308 */
 309static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
 310{
 311	u64 interval;
 312	u64 tmp, ntpinterval;
 313	struct clocksource *old_clock;
 314
 315	++tk->cs_was_changed_seq;
 316	old_clock = tk->tkr_mono.clock;
 317	tk->tkr_mono.clock = clock;
 318	tk->tkr_mono.mask = clock->mask;
 319	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
 320
 321	tk->tkr_raw.clock = clock;
 322	tk->tkr_raw.mask = clock->mask;
 323	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
 324
 325	/* Do the ns -> cycle conversion first, using original mult */
 326	tmp = NTP_INTERVAL_LENGTH;
 327	tmp <<= clock->shift;
 328	ntpinterval = tmp;
 329	tmp += clock->mult/2;
 330	do_div(tmp, clock->mult);
 331	if (tmp == 0)
 332		tmp = 1;
 333
 334	interval = (u64) tmp;
 335	tk->cycle_interval = interval;
 336
 337	/* Go back from cycles -> shifted ns */
 338	tk->xtime_interval = interval * clock->mult;
 339	tk->xtime_remainder = ntpinterval - tk->xtime_interval;
 340	tk->raw_interval = interval * clock->mult;
 341
 342	 /* if changing clocks, convert xtime_nsec shift units */
 343	if (old_clock) {
 344		int shift_change = clock->shift - old_clock->shift;
 345		if (shift_change < 0) {
 346			tk->tkr_mono.xtime_nsec >>= -shift_change;
 347			tk->tkr_raw.xtime_nsec >>= -shift_change;
 348		} else {
 349			tk->tkr_mono.xtime_nsec <<= shift_change;
 350			tk->tkr_raw.xtime_nsec <<= shift_change;
 351		}
 352	}
 353
 354	tk->tkr_mono.shift = clock->shift;
 355	tk->tkr_raw.shift = clock->shift;
 356
 357	tk->ntp_error = 0;
 358	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
 359	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
 360
 361	/*
 362	 * The timekeeper keeps its own mult values for the currently
 363	 * active clocksource. These value will be adjusted via NTP
 364	 * to counteract clock drifting.
 365	 */
 366	tk->tkr_mono.mult = clock->mult;
 367	tk->tkr_raw.mult = clock->mult;
 368	tk->ntp_err_mult = 0;
 369	tk->skip_second_overflow = 0;
 370}
 371
 372/* Timekeeper helper functions. */
 373static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta)
 374{
 375	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);
 376}
 377
 378static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
 379{
 380	/* Calculate the delta since the last update_wall_time() */
 381	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;
 382
 383	/*
 384	 * This detects both negative motion and the case where the delta
 385	 * overflows the multiplication with tkr->mult.
 386	 */
 387	if (unlikely(delta > tkr->clock->max_cycles)) {
 388		/*
 389		 * Handle clocksource inconsistency between CPUs to prevent
 390		 * time from going backwards by checking for the MSB of the
 391		 * mask being set in the delta.
 392		 */
 393		if (delta & ~(mask >> 1))
 394			return tkr->xtime_nsec >> tkr->shift;
 395
 396		return delta_to_ns_safe(tkr, delta);
 397	}
 398
 399	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;
 400}
 401
 402static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
 403{
 404	return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr));
 405}
 406
 407/**
 408 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
 409 * @tkr: Timekeeping readout base from which we take the update
 410 * @tkf: Pointer to NMI safe timekeeper
 411 *
 412 * We want to use this from any context including NMI and tracing /
 413 * instrumenting the timekeeping code itself.
 414 *
 415 * Employ the latch technique; see @write_seqcount_latch.
 416 *
 417 * So if a NMI hits the update of base[0] then it will use base[1]
 418 * which is still consistent. In the worst case this can result is a
 419 * slightly wrong timestamp (a few nanoseconds). See
 420 * @ktime_get_mono_fast_ns.
 421 */
 422static void update_fast_timekeeper(const struct tk_read_base *tkr,
 423				   struct tk_fast *tkf)
 424{
 425	struct tk_read_base *base = tkf->base;
 426
 427	/* Force readers off to base[1] */
 428	write_seqcount_latch_begin(&tkf->seq);
 429
 430	/* Update base[0] */
 431	memcpy(base, tkr, sizeof(*base));
 432
 433	/* Force readers back to base[0] */
 434	write_seqcount_latch(&tkf->seq);
 435
 436	/* Update base[1] */
 437	memcpy(base + 1, base, sizeof(*base));
 438
 439	write_seqcount_latch_end(&tkf->seq);
 440}
 441
 442static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
 443{
 444	struct tk_read_base *tkr;
 445	unsigned int seq;
 446	u64 now;
 447
 448	do {
 449		seq = read_seqcount_latch(&tkf->seq);
 450		tkr = tkf->base + (seq & 0x01);
 451		now = ktime_to_ns(tkr->base);
 452		now += timekeeping_get_ns(tkr);
 453	} while (read_seqcount_latch_retry(&tkf->seq, seq));
 454
 455	return now;
 456}
 457
 458/**
 459 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
 460 *
 461 * This timestamp is not guaranteed to be monotonic across an update.
 462 * The timestamp is calculated by:
 463 *
 464 *	now = base_mono + clock_delta * slope
 465 *
 466 * So if the update lowers the slope, readers who are forced to the
 467 * not yet updated second array are still using the old steeper slope.
 468 *
 469 * tmono
 470 * ^
 471 * |    o  n
 472 * |   o n
 473 * |  u
 474 * | o
 475 * |o
 476 * |12345678---> reader order
 477 *
 478 * o = old slope
 479 * u = update
 480 * n = new slope
 481 *
 482 * So reader 6 will observe time going backwards versus reader 5.
 483 *
 484 * While other CPUs are likely to be able to observe that, the only way
 485 * for a CPU local observation is when an NMI hits in the middle of
 486 * the update. Timestamps taken from that NMI context might be ahead
 487 * of the following timestamps. Callers need to be aware of that and
 488 * deal with it.
 489 */
 490u64 notrace ktime_get_mono_fast_ns(void)
 491{
 492	return __ktime_get_fast_ns(&tk_fast_mono);
 493}
 494EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
 495
 496/**
 497 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
 498 *
 499 * Contrary to ktime_get_mono_fast_ns() this is always correct because the
 500 * conversion factor is not affected by NTP/PTP correction.
 501 */
 502u64 notrace ktime_get_raw_fast_ns(void)
 503{
 504	return __ktime_get_fast_ns(&tk_fast_raw);
 505}
 506EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
 507
 508/**
 509 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
 510 *
 511 * To keep it NMI safe since we're accessing from tracing, we're not using a
 512 * separate timekeeper with updates to monotonic clock and boot offset
 513 * protected with seqcounts. This has the following minor side effects:
 514 *
 515 * (1) Its possible that a timestamp be taken after the boot offset is updated
 516 * but before the timekeeper is updated. If this happens, the new boot offset
 517 * is added to the old timekeeping making the clock appear to update slightly
 518 * earlier:
 519 *    CPU 0                                        CPU 1
 520 *    timekeeping_inject_sleeptime64()
 521 *    __timekeeping_inject_sleeptime(tk, delta);
 522 *                                                 timestamp();
 523 *    timekeeping_update_staged(tkd, TK_CLEAR_NTP...);
 524 *
 525 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
 526 * partially updated.  Since the tk->offs_boot update is a rare event, this
 527 * should be a rare occurrence which postprocessing should be able to handle.
 528 *
 529 * The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns()
 530 * apply as well.
 531 */
 532u64 notrace ktime_get_boot_fast_ns(void)
 533{
 534	struct timekeeper *tk = &tk_core.timekeeper;
 535
 536	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));
 537}
 538EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
 539
 540/**
 541 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
 542 *
 543 * The same limitations as described for ktime_get_boot_fast_ns() apply. The
 544 * mono time and the TAI offset are not read atomically which may yield wrong
 545 * readouts. However, an update of the TAI offset is an rare event e.g., caused
 546 * by settime or adjtimex with an offset. The user of this function has to deal
 547 * with the possibility of wrong timestamps in post processing.
 548 */
 549u64 notrace ktime_get_tai_fast_ns(void)
 550{
 551	struct timekeeper *tk = &tk_core.timekeeper;
 552
 553	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));
 554}
 555EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns);
 556
 557/**
 558 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
 559 *
 560 * See ktime_get_mono_fast_ns() for documentation of the time stamp ordering.
 561 */
 562u64 ktime_get_real_fast_ns(void)
 563{
 564	struct tk_fast *tkf = &tk_fast_mono;
 565	struct tk_read_base *tkr;
 566	u64 baser, delta;
 567	unsigned int seq;
 568
 569	do {
 570		seq = raw_read_seqcount_latch(&tkf->seq);
 571		tkr = tkf->base + (seq & 0x01);
 572		baser = ktime_to_ns(tkr->base_real);
 573		delta = timekeeping_get_ns(tkr);
 574	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));
 575
 576	return baser + delta;
 577}
 578EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
 579
 580/**
 581 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
 582 * @tk: Timekeeper to snapshot.
 583 *
 584 * It generally is unsafe to access the clocksource after timekeeping has been
 585 * suspended, so take a snapshot of the readout base of @tk and use it as the
 586 * fast timekeeper's readout base while suspended.  It will return the same
 587 * number of cycles every time until timekeeping is resumed at which time the
 588 * proper readout base for the fast timekeeper will be restored automatically.
 589 */
 590static void halt_fast_timekeeper(const struct timekeeper *tk)
 591{
 592	static struct tk_read_base tkr_dummy;
 593	const struct tk_read_base *tkr = &tk->tkr_mono;
 594
 595	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
 596	cycles_at_suspend = tk_clock_read(tkr);
 597	tkr_dummy.clock = &dummy_clock;
 598	tkr_dummy.base_real = tkr->base + tk->offs_real;
 599	update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
 600
 601	tkr = &tk->tkr_raw;
 602	memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
 603	tkr_dummy.clock = &dummy_clock;
 604	update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
 605}
 606
 607static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
 608
 609static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
 610{
 611	raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
 612}
 613
 614/**
 615 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
 616 * @nb: Pointer to the notifier block to register
 617 */
 618int pvclock_gtod_register_notifier(struct notifier_block *nb)
 619{
 620	struct timekeeper *tk = &tk_core.timekeeper;
 621	int ret;
 622
 623	guard(raw_spinlock_irqsave)(&tk_core.lock);
 624	ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
 625	update_pvclock_gtod(tk, true);
 626
 627	return ret;
 628}
 629EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
 630
 631/**
 632 * pvclock_gtod_unregister_notifier - unregister a pvclock
 633 * timedata update listener
 634 * @nb: Pointer to the notifier block to unregister
 635 */
 636int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
 637{
 638	guard(raw_spinlock_irqsave)(&tk_core.lock);
 639	return raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
 640}
 641EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
 642
 643/*
 644 * tk_update_leap_state - helper to update the next_leap_ktime
 645 */
 646static inline void tk_update_leap_state(struct timekeeper *tk)
 647{
 648	tk->next_leap_ktime = ntp_get_next_leap(tk->id);
 649	if (tk->next_leap_ktime != KTIME_MAX)
 650		/* Convert to monotonic time */
 651		tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
 652}
 653
 654/*
 655 * Leap state update for both shadow and the real timekeeper
 656 * Separate to spare a full memcpy() of the timekeeper.
 657 */
 658static void tk_update_leap_state_all(struct tk_data *tkd)
 659{
 660	write_seqcount_begin(&tkd->seq);
 661	tk_update_leap_state(&tkd->shadow_timekeeper);
 662	tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;
 663	write_seqcount_end(&tkd->seq);
 664}
 665
 666/*
 667 * Update the ktime_t based scalar nsec members of the timekeeper
 668 */
 669static inline void tk_update_ktime_data(struct timekeeper *tk)
 670{
 671	u64 seconds;
 672	u32 nsec;
 673
 674	/*
 675	 * The xtime based monotonic readout is:
 676	 *	nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
 677	 * The ktime based monotonic readout is:
 678	 *	nsec = base_mono + now();
 679	 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
 680	 */
 681	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
 682	nsec = (u32) tk->wall_to_monotonic.tv_nsec;
 683	tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
 684
 685	/*
 686	 * The sum of the nanoseconds portions of xtime and
 687	 * wall_to_monotonic can be greater/equal one second. Take
 688	 * this into account before updating tk->ktime_sec.
 689	 */
 690	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
 691	if (nsec >= NSEC_PER_SEC)
 692		seconds++;
 693	tk->ktime_sec = seconds;
 694
 695	/* Update the monotonic raw base */
 696	tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
 697}
 698
 699/*
 700 * Restore the shadow timekeeper from the real timekeeper.
 701 */
 702static void timekeeping_restore_shadow(struct tk_data *tkd)
 703{
 704	lockdep_assert_held(&tkd->lock);
 705	memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper));
 706}
 707
 708static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action)
 709{
 710	struct timekeeper *tk = &tkd->shadow_timekeeper;
 711
 712	lockdep_assert_held(&tkd->lock);
 713
 714	/*
 715	 * Block out readers before running the updates below because that
 716	 * updates VDSO and other time related infrastructure. Not blocking
 717	 * the readers might let a reader see time going backwards when
 718	 * reading from the VDSO after the VDSO update and then reading in
 719	 * the kernel from the timekeeper before that got updated.
 720	 */
 721	write_seqcount_begin(&tkd->seq);
 722
 723	if (action & TK_CLEAR_NTP) {
 724		tk->ntp_error = 0;
 725		ntp_clear(tk->id);
 726	}
 727
 728	tk_update_leap_state(tk);
 729	tk_update_ktime_data(tk);
 730	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
 731
 732	if (tk->id == TIMEKEEPER_CORE) {
 733		update_vsyscall(tk);
 734		update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
 735
 736		update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
 737		update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);
 738	} else if (tk_is_aux(tk)) {
 739		vdso_time_update_aux(tk);
 740	}
 741
 742	if (action & TK_CLOCK_WAS_SET)
 743		tk->clock_was_set_seq++;
 744
 745	/*
 746	 * Update the real timekeeper.
 747	 *
 748	 * We could avoid this memcpy() by switching pointers, but that has
 749	 * the downside that the reader side does not longer benefit from
 750	 * the cacheline optimized data layout of the timekeeper and requires
 751	 * another indirection.
 752	 */
 753	memcpy(&tkd->timekeeper, tk, sizeof(*tk));
 754	write_seqcount_end(&tkd->seq);
 755}
 756
 757/**
 758 * timekeeping_forward_now - update clock to the current time
 759 * @tk:		Pointer to the timekeeper to update
 760 *
 761 * Forward the current clock to update its state since the last call to
 762 * update_wall_time(). This is useful before significant clock changes,
 763 * as it avoids having to deal with this time offset explicitly.
 764 */
 765static void timekeeping_forward_now(struct timekeeper *tk)
 766{
 767	u64 cycle_now, delta;
 768
 769	cycle_now = tk_clock_read(&tk->tkr_mono);
 770	delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask,
 771				  tk->tkr_mono.clock->max_raw_delta);
 772	tk->tkr_mono.cycle_last = cycle_now;
 773	tk->tkr_raw.cycle_last  = cycle_now;
 774
 775	while (delta > 0) {
 776		u64 max = tk->tkr_mono.clock->max_cycles;
 777		u64 incr = delta < max ? delta : max;
 778
 779		tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;
 780		tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;
 781		tk_normalize_xtime(tk);
 782		delta -= incr;
 783	}
 784	tk_update_coarse_nsecs(tk);
 785}
 786
 787/**
 788 * ktime_get_real_ts64 - Returns the time of day in a timespec64.
 789 * @ts:		pointer to the timespec to be set
 790 *
 791 * Returns the time of day in a timespec64 (WARN if suspended).
 792 */
 793void ktime_get_real_ts64(struct timespec64 *ts)
 794{
 795	struct timekeeper *tk = &tk_core.timekeeper;
 796	unsigned int seq;
 797	u64 nsecs;
 798
 799	WARN_ON(timekeeping_suspended);
 800
 801	do {
 802		seq = read_seqcount_begin(&tk_core.seq);
 803
 804		ts->tv_sec = tk->xtime_sec;
 805		nsecs = timekeeping_get_ns(&tk->tkr_mono);
 806
 807	} while (read_seqcount_retry(&tk_core.seq, seq));
 808
 809	ts->tv_nsec = 0;
 810	timespec64_add_ns(ts, nsecs);
 811}
 812EXPORT_SYMBOL(ktime_get_real_ts64);
 813
 814ktime_t ktime_get(void)
 815{
 816	struct timekeeper *tk = &tk_core.timekeeper;
 817	unsigned int seq;
 818	ktime_t base;
 819	u64 nsecs;
 820
 821	WARN_ON(timekeeping_suspended);
 822
 823	do {
 824		seq = read_seqcount_begin(&tk_core.seq);
 825		base = tk->tkr_mono.base;
 826		nsecs = timekeeping_get_ns(&tk->tkr_mono);
 827
 828	} while (read_seqcount_retry(&tk_core.seq, seq));
 829
 830	return ktime_add_ns(base, nsecs);
 831}
 832EXPORT_SYMBOL_GPL(ktime_get);
 833
 834u32 ktime_get_resolution_ns(void)
 835{
 836	struct timekeeper *tk = &tk_core.timekeeper;
 837	unsigned int seq;
 838	u32 nsecs;
 839
 840	WARN_ON(timekeeping_suspended);
 841
 842	do {
 843		seq = read_seqcount_begin(&tk_core.seq);
 844		nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
 845	} while (read_seqcount_retry(&tk_core.seq, seq));
 846
 847	return nsecs;
 848}
 849EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
 850
 851static ktime_t *offsets[TK_OFFS_MAX] = {
 852	[TK_OFFS_REAL]	= &tk_core.timekeeper.offs_real,
 853	[TK_OFFS_BOOT]	= &tk_core.timekeeper.offs_boot,
 854	[TK_OFFS_TAI]	= &tk_core.timekeeper.offs_tai,
 855};
 856
 857ktime_t ktime_get_with_offset(enum tk_offsets offs)
 858{
 859	struct timekeeper *tk = &tk_core.timekeeper;
 860	unsigned int seq;
 861	ktime_t base, *offset = offsets[offs];
 862	u64 nsecs;
 863
 864	WARN_ON(timekeeping_suspended);
 865
 866	do {
 867		seq = read_seqcount_begin(&tk_core.seq);
 868		base = ktime_add(tk->tkr_mono.base, *offset);
 869		nsecs = timekeeping_get_ns(&tk->tkr_mono);
 870
 871	} while (read_seqcount_retry(&tk_core.seq, seq));
 872
 873	return ktime_add_ns(base, nsecs);
 874
 875}
 876EXPORT_SYMBOL_GPL(ktime_get_with_offset);
 877
 878ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
 879{
 880	struct timekeeper *tk = &tk_core.timekeeper;
 881	ktime_t base, *offset = offsets[offs];
 882	unsigned int seq;
 883	u64 nsecs;
 884
 885	WARN_ON(timekeeping_suspended);
 886
 887	do {
 888		seq = read_seqcount_begin(&tk_core.seq);
 889		base = ktime_add(tk->tkr_mono.base, *offset);
 890		nsecs = tk->coarse_nsec;
 891
 892	} while (read_seqcount_retry(&tk_core.seq, seq));
 893
 894	return ktime_add_ns(base, nsecs);
 895}
 896EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
 897
 898/**
 899 * ktime_mono_to_any() - convert monotonic time to any other time
 900 * @tmono:	time to convert.
 901 * @offs:	which offset to use
 902 */
 903ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
 904{
 905	ktime_t *offset = offsets[offs];
 906	unsigned int seq;
 907	ktime_t tconv;
 908
 909	if (IS_ENABLED(CONFIG_64BIT)) {
 910		/*
 911		 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and
 912		 * tk_update_sleep_time().
 913		 */
 914		return ktime_add(tmono, READ_ONCE(*offset));
 915	}
 916
 917	do {
 918		seq = read_seqcount_begin(&tk_core.seq);
 919		tconv = ktime_add(tmono, *offset);
 920	} while (read_seqcount_retry(&tk_core.seq, seq));
 921
 922	return tconv;
 923}
 924EXPORT_SYMBOL_GPL(ktime_mono_to_any);
 925
 926/**
 927 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
 928 */
 929ktime_t ktime_get_raw(void)
 930{
 931	struct timekeeper *tk = &tk_core.timekeeper;
 932	unsigned int seq;
 933	ktime_t base;
 934	u64 nsecs;
 935
 936	do {
 937		seq = read_seqcount_begin(&tk_core.seq);
 938		base = tk->tkr_raw.base;
 939		nsecs = timekeeping_get_ns(&tk->tkr_raw);
 940
 941	} while (read_seqcount_retry(&tk_core.seq, seq));
 942
 943	return ktime_add_ns(base, nsecs);
 944}
 945EXPORT_SYMBOL_GPL(ktime_get_raw);
 946
 947/**
 948 * ktime_get_ts64 - get the monotonic clock in timespec64 format
 949 * @ts:		pointer to timespec variable
 950 *
 951 * The function calculates the monotonic clock from the realtime
 952 * clock and the wall_to_monotonic offset and stores the result
 953 * in normalized timespec64 format in the variable pointed to by @ts.
 954 */
 955void ktime_get_ts64(struct timespec64 *ts)
 956{
 957	struct timekeeper *tk = &tk_core.timekeeper;
 958	struct timespec64 tomono;
 959	unsigned int seq;
 960	u64 nsec;
 961
 962	WARN_ON(timekeeping_suspended);
 963
 964	do {
 965		seq = read_seqcount_begin(&tk_core.seq);
 966		ts->tv_sec = tk->xtime_sec;
 967		nsec = timekeeping_get_ns(&tk->tkr_mono);
 968		tomono = tk->wall_to_monotonic;
 969
 970	} while (read_seqcount_retry(&tk_core.seq, seq));
 971
 972	ts->tv_sec += tomono.tv_sec;
 973	ts->tv_nsec = 0;
 974	timespec64_add_ns(ts, nsec + tomono.tv_nsec);
 975}
 976EXPORT_SYMBOL_GPL(ktime_get_ts64);
 977
 978/**
 979 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
 980 *
 981 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
 982 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
 983 * works on both 32 and 64 bit systems. On 32 bit systems the readout
 984 * covers ~136 years of uptime which should be enough to prevent
 985 * premature wrap arounds.
 986 */
 987time64_t ktime_get_seconds(void)
 988{
 989	struct timekeeper *tk = &tk_core.timekeeper;
 990
 991	WARN_ON(timekeeping_suspended);
 992	return tk->ktime_sec;
 993}
 994EXPORT_SYMBOL_GPL(ktime_get_seconds);
 995
 996/**
 997 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
 998 *
 999 * Returns the wall clock seconds since 1970.
1000 *
1001 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
1002 * 32bit systems the access must be protected with the sequence
1003 * counter to provide "atomic" access to the 64bit tk->xtime_sec
1004 * value.
1005 */
1006time64_t ktime_get_real_seconds(void)
1007{
1008	struct timekeeper *tk = &tk_core.timekeeper;
1009	time64_t seconds;
1010	unsigned int seq;
1011
1012	if (IS_ENABLED(CONFIG_64BIT))
1013		return tk->xtime_sec;
1014
1015	do {
1016		seq = read_seqcount_begin(&tk_core.seq);
1017		seconds = tk->xtime_sec;
1018
1019	} while (read_seqcount_retry(&tk_core.seq, seq));
1020
1021	return seconds;
1022}
1023EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
1024
1025/**
1026 * __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds
1027 *
1028 * The same as ktime_get_real_seconds() but without the sequence counter
1029 * protection. This function is used in restricted contexts like the x86 MCE
1030 * handler and in KGDB. It's unprotected on 32-bit vs. concurrent half
1031 * completed modification and only to be used for such critical contexts.
1032 *
1033 * Returns: Racy snapshot of the CLOCK_REALTIME seconds value
1034 */
1035noinstr time64_t __ktime_get_real_seconds(void)
1036{
1037	struct timekeeper *tk = &tk_core.timekeeper;
1038
1039	return tk->xtime_sec;
1040}
1041
1042/**
1043 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1044 * @systime_snapshot:	pointer to struct receiving the system time snapshot
1045 */
1046void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
1047{
1048	struct timekeeper *tk = &tk_core.timekeeper;
1049	unsigned int seq;
1050	ktime_t base_raw;
1051	ktime_t base_real;
1052	ktime_t base_boot;
1053	u64 nsec_raw;
1054	u64 nsec_real;
1055	u64 now;
1056
1057	WARN_ON_ONCE(timekeeping_suspended);
1058
1059	do {
1060		seq = read_seqcount_begin(&tk_core.seq);
1061		now = tk_clock_read(&tk->tkr_mono);
1062		systime_snapshot->cs_id = tk->tkr_mono.clock->id;
1063		systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
1064		systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
1065		base_real = ktime_add(tk->tkr_mono.base,
1066				      tk_core.timekeeper.offs_real);
1067		base_boot = ktime_add(tk->tkr_mono.base,
1068				      tk_core.timekeeper.offs_boot);
1069		base_raw = tk->tkr_raw.base;
1070		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
1071		nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
1072	} while (read_seqcount_retry(&tk_core.seq, seq));
1073
1074	systime_snapshot->cycles = now;
1075	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
1076	systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);
1077	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
1078}
1079EXPORT_SYMBOL_GPL(ktime_get_snapshot);
1080
1081/* Scale base by mult/div checking for overflow */
1082static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
1083{
1084	u64 tmp, rem;
1085
1086	tmp = div64_u64_rem(*base, div, &rem);
1087
1088	if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
1089	    ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1090		return -EOVERFLOW;
1091	tmp *= mult;
1092
1093	rem = div64_u64(rem * mult, div);
1094	*base = tmp + rem;
1095	return 0;
1096}
1097
1098/**
1099 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1100 * @history:			Snapshot representing start of history
1101 * @partial_history_cycles:	Cycle offset into history (fractional part)
1102 * @total_history_cycles:	Total history length in cycles
1103 * @discontinuity:		True indicates clock was set on history period
1104 * @ts:				Cross timestamp that should be adjusted using
1105 *	partial/total ratio
1106 *
1107 * Helper function used by get_device_system_crosststamp() to correct the
1108 * crosstimestamp corresponding to the start of the current interval to the
1109 * system counter value (timestamp point) provided by the driver. The
1110 * total_history_* quantities are the total history starting at the provided
1111 * reference point and ending at the start of the current interval. The cycle
1112 * count between the driver timestamp point and the start of the current
1113 * interval is partial_history_cycles.
1114 */
1115static int adjust_historical_crosststamp(struct system_time_snapshot *history,
1116					 u64 partial_history_cycles,
1117					 u64 total_history_cycles,
1118					 bool discontinuity,
1119					 struct system_device_crosststamp *ts)
1120{
1121	struct timekeeper *tk = &tk_core.timekeeper;
1122	u64 corr_raw, corr_real;
1123	bool interp_forward;
1124	int ret;
1125
1126	if (total_history_cycles == 0 || partial_history_cycles == 0)
1127		return 0;
1128
1129	/* Interpolate shortest distance from beginning or end of history */
1130	interp_forward = partial_history_cycles > total_history_cycles / 2;
1131	partial_history_cycles = interp_forward ?
1132		total_history_cycles - partial_history_cycles :
1133		partial_history_cycles;
1134
1135	/*
1136	 * Scale the monotonic raw time delta by:
1137	 *	partial_history_cycles / total_history_cycles
1138	 */
1139	corr_raw = (u64)ktime_to_ns(
1140		ktime_sub(ts->sys_monoraw, history->raw));
1141	ret = scale64_check_overflow(partial_history_cycles,
1142				     total_history_cycles, &corr_raw);
1143	if (ret)
1144		return ret;
1145
1146	/*
1147	 * If there is a discontinuity in the history, scale monotonic raw
1148	 *	correction by:
1149	 *	mult(real)/mult(raw) yielding the realtime correction
1150	 * Otherwise, calculate the realtime correction similar to monotonic
1151	 *	raw calculation
1152	 */
1153	if (discontinuity) {
1154		corr_real = mul_u64_u32_div
1155			(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1156	} else {
1157		corr_real = (u64)ktime_to_ns(
1158			ktime_sub(ts->sys_realtime, history->real));
1159		ret = scale64_check_overflow(partial_history_cycles,
1160					     total_history_cycles, &corr_real);
1161		if (ret)
1162			return ret;
1163	}
1164
1165	/* Fixup monotonic raw and real time time values */
1166	if (interp_forward) {
1167		ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1168		ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1169	} else {
1170		ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1171		ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1172	}
1173
1174	return 0;
1175}
1176
1177/*
1178 * timestamp_in_interval - true if ts is chronologically in [start, end]
1179 *
1180 * True if ts occurs chronologically at or after start, and before or at end.
1181 */
1182static bool timestamp_in_interval(u64 start, u64 end, u64 ts)
1183{
1184	if (ts >= start && ts <= end)
1185		return true;
1186	if (start > end && (ts >= start || ts <= end))
1187		return true;
1188	return false;
1189}
1190
1191static bool convert_clock(u64 *val, u32 numerator, u32 denominator)
1192{
1193	u64 rem, res;
1194
1195	if (!numerator || !denominator)
1196		return false;
1197
1198	res = div64_u64_rem(*val, denominator, &rem) * numerator;
1199	*val = res + div_u64(rem * numerator, denominator);
1200	return true;
1201}
1202
1203static bool convert_base_to_cs(struct system_counterval_t *scv)
1204{
1205	struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
1206	struct clocksource_base *base;
1207	u32 num, den;
1208
1209	/* The timestamp was taken from the time keeper clock source */
1210	if (cs->id == scv->cs_id)
1211		return true;
1212
1213	/*
1214	 * Check whether cs_id matches the base clock. Prevent the compiler from
1215	 * re-evaluating @base as the clocksource might change concurrently.
1216	 */
1217	base = READ_ONCE(cs->base);
1218	if (!base || base->id != scv->cs_id)
1219		return false;
1220
1221	num = scv->use_nsecs ? cs->freq_khz : base->numerator;
1222	den = scv->use_nsecs ? USEC_PER_SEC : base->denominator;
1223
1224	if (!convert_clock(&scv->cycles, num, den))
1225		return false;
1226
1227	scv->cycles += base->offset;
1228	return true;
1229}
1230
1231static bool convert_cs_to_base(u64 *cycles, enum clocksource_ids base_id)
1232{
1233	struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock;
1234	struct clocksource_base *base;
1235
1236	/*
1237	 * Check whether base_id matches the base clock. Prevent the compiler from
1238	 * re-evaluating @base as the clocksource might change concurrently.
1239	 */
1240	base = READ_ONCE(cs->base);
1241	if (!base || base->id != base_id)
1242		return false;
1243
1244	*cycles -= base->offset;
1245	if (!convert_clock(cycles, base->denominator, base->numerator))
1246		return false;
1247	return true;
1248}
1249
1250static bool convert_ns_to_cs(u64 *delta)
1251{
1252	struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
1253
1254	if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta))
1255		return false;
1256
1257	*delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult);
1258	return true;
1259}
1260
1261/**
1262 * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp
1263 * @treal:	CLOCK_REALTIME timestamp to convert
1264 * @base_id:	base clocksource id
1265 * @cycles:	pointer to store the converted base clock timestamp
1266 *
1267 * Converts a supplied, future realtime clock value to the corresponding base clock value.
1268 *
1269 * Return:  true if the conversion is successful, false otherwise.
1270 */
1271bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles)
1272{
1273	struct timekeeper *tk = &tk_core.timekeeper;
1274	unsigned int seq;
1275	u64 delta;
1276
1277	do {
1278		seq = read_seqcount_begin(&tk_core.seq);
1279		if ((u64)treal < tk->tkr_mono.base_real)
1280			return false;
1281		delta = (u64)treal - tk->tkr_mono.base_real;
1282		if (!convert_ns_to_cs(&delta))
1283			return false;
1284		*cycles = tk->tkr_mono.cycle_last + delta;
1285		if (!convert_cs_to_base(cycles, base_id))
1286			return false;
1287	} while (read_seqcount_retry(&tk_core.seq, seq));
1288
1289	return true;
1290}
1291EXPORT_SYMBOL_GPL(ktime_real_to_base_clock);
1292
1293/**
1294 * get_device_system_crosststamp - Synchronously capture system/device timestamp
1295 * @get_time_fn:	Callback to get simultaneous device time and
1296 *	system counter from the device driver
1297 * @ctx:		Context passed to get_time_fn()
1298 * @history_begin:	Historical reference point used to interpolate system
1299 *	time when counter provided by the driver is before the current interval
1300 * @xtstamp:		Receives simultaneously captured system and device time
1301 *
1302 * Reads a timestamp from a device and correlates it to system time
1303 */
1304int get_device_system_crosststamp(int (*get_time_fn)
1305				  (ktime_t *device_time,
1306				   struct system_counterval_t *sys_counterval,
1307				   void *ctx),
1308				  void *ctx,
1309				  struct system_time_snapshot *history_begin,
1310				  struct system_device_crosststamp *xtstamp)
1311{
1312	struct system_counterval_t system_counterval = {};
1313	struct timekeeper *tk = &tk_core.timekeeper;
1314	u64 cycles, now, interval_start;
1315	unsigned int clock_was_set_seq = 0;
1316	ktime_t base_real, base_raw;
1317	u64 nsec_real, nsec_raw;
1318	u8 cs_was_changed_seq;
1319	unsigned int seq;
1320	bool do_interp;
1321	int ret;
1322
1323	do {
1324		seq = read_seqcount_begin(&tk_core.seq);
1325		/*
1326		 * Try to synchronously capture device time and a system
1327		 * counter value calling back into the device driver
1328		 */
1329		ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1330		if (ret)
1331			return ret;
1332
1333		/*
1334		 * Verify that the clocksource ID associated with the captured
1335		 * system counter value is the same as for the currently
1336		 * installed timekeeper clocksource
1337		 */
1338		if (system_counterval.cs_id == CSID_GENERIC ||
1339		    !convert_base_to_cs(&system_counterval))
1340			return -ENODEV;
1341		cycles = system_counterval.cycles;
1342
1343		/*
1344		 * Check whether the system counter value provided by the
1345		 * device driver is on the current timekeeping interval.
1346		 */
1347		now = tk_clock_read(&tk->tkr_mono);
1348		interval_start = tk->tkr_mono.cycle_last;
1349		if (!timestamp_in_interval(interval_start, now, cycles)) {
1350			clock_was_set_seq = tk->clock_was_set_seq;
1351			cs_was_changed_seq = tk->cs_was_changed_seq;
1352			cycles = interval_start;
1353			do_interp = true;
1354		} else {
1355			do_interp = false;
1356		}
1357
1358		base_real = ktime_add(tk->tkr_mono.base,
1359				      tk_core.timekeeper.offs_real);
1360		base_raw = tk->tkr_raw.base;
1361
1362		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);
1363		nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);
1364	} while (read_seqcount_retry(&tk_core.seq, seq));
1365
1366	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1367	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
1368
1369	/*
1370	 * Interpolate if necessary, adjusting back from the start of the
1371	 * current interval
1372	 */
1373	if (do_interp) {
1374		u64 partial_history_cycles, total_history_cycles;
1375		bool discontinuity;
1376
1377		/*
1378		 * Check that the counter value is not before the provided
1379		 * history reference and that the history doesn't cross a
1380		 * clocksource change
1381		 */
1382		if (!history_begin ||
1383		    !timestamp_in_interval(history_begin->cycles,
1384					   cycles, system_counterval.cycles) ||
1385		    history_begin->cs_was_changed_seq != cs_was_changed_seq)
1386			return -EINVAL;
1387		partial_history_cycles = cycles - system_counterval.cycles;
1388		total_history_cycles = cycles - history_begin->cycles;
1389		discontinuity =
1390			history_begin->clock_was_set_seq != clock_was_set_seq;
1391
1392		ret = adjust_historical_crosststamp(history_begin,
1393						    partial_history_cycles,
1394						    total_history_cycles,
1395						    discontinuity, xtstamp);
1396		if (ret)
1397			return ret;
1398	}
1399
1400	return 0;
1401}
1402EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1403
1404/**
1405 * timekeeping_clocksource_has_base - Check whether the current clocksource
1406 *				      is based on given a base clock
1407 * @id:		base clocksource ID
1408 *
1409 * Note:	The return value is a snapshot which can become invalid right
1410 *		after the function returns.
1411 *
1412 * Return:	true if the timekeeper clocksource has a base clock with @id,
1413 *		false otherwise
1414 */
1415bool timekeeping_clocksource_has_base(enum clocksource_ids id)
1416{
1417	/*
1418	 * This is a snapshot, so no point in using the sequence
1419	 * count. Just prevent the compiler from re-evaluating @base as the
1420	 * clocksource might change concurrently.
1421	 */
1422	struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base);
1423
1424	return base ? base->id == id : false;
1425}
1426EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base);
1427
1428/**
1429 * do_settimeofday64 - Sets the time of day.
1430 * @ts:     pointer to the timespec64 variable containing the new time
1431 *
1432 * Sets the time of day to the new time and update NTP and notify hrtimers
1433 */
1434int do_settimeofday64(const struct timespec64 *ts)
1435{
1436	struct timespec64 ts_delta, xt;
1437
1438	if (!timespec64_valid_settod(ts))
1439		return -EINVAL;
1440
1441	scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
1442		struct timekeeper *tks = &tk_core.shadow_timekeeper;
1443
1444		timekeeping_forward_now(tks);
1445
1446		xt = tk_xtime(tks);
1447		ts_delta = timespec64_sub(*ts, xt);
1448
1449		if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) {
1450			timekeeping_restore_shadow(&tk_core);
1451			return -EINVAL;
1452		}
1453
1454		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta));
1455		tk_set_xtime(tks, ts);
1456		timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
1457	}
1458
1459	/* Signal hrtimers about time change */
1460	clock_was_set(CLOCK_SET_WALL);
1461
1462	audit_tk_injoffset(ts_delta);
1463	add_device_randomness(ts, sizeof(*ts));
1464	return 0;
1465}
1466EXPORT_SYMBOL(do_settimeofday64);
1467
1468static inline bool timekeeper_is_core_tk(struct timekeeper *tk)
1469{
1470	return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE;
1471}
1472
1473/**
1474 * __timekeeping_inject_offset - Adds or subtracts from the current time.
1475 * @tkd:	Pointer to the timekeeper to modify
1476 * @ts:		Pointer to the timespec variable containing the offset
1477 *
1478 * Adds or subtracts an offset value from the current time.
1479 */
1480static int __timekeeping_inject_offset(struct tk_data *tkd, const struct timespec64 *ts)
1481{
1482	struct timekeeper *tks = &tkd->shadow_timekeeper;
1483	struct timespec64 tmp;
1484
1485	if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
1486		return -EINVAL;
1487
1488	timekeeping_forward_now(tks);
1489
1490	if (timekeeper_is_core_tk(tks)) {
1491		/* Make sure the proposed value is valid */
1492		tmp = timespec64_add(tk_xtime(tks), *ts);
1493		if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||
1494		    !timespec64_valid_settod(&tmp)) {
1495			timekeeping_restore_shadow(tkd);
1496			return -EINVAL;
1497		}
1498
1499		tk_xtime_add(tks, ts);
1500		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));
1501	} else {
1502		struct tk_read_base *tkr_mono = &tks->tkr_mono;
1503		ktime_t now, offs;
1504
1505		/* Get the current time */
1506		now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono));
1507		/* Add the relative offset change */
1508		offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts));
1509
1510		/* Prevent that the resulting time becomes negative */
1511		if (ktime_add(now, offs) < 0) {
1512			timekeeping_restore_shadow(tkd);
1513			return -EINVAL;
1514		}
1515		tk_update_aux_offs(tks, offs);
1516	}
1517
1518	timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
1519	return 0;
1520}
1521
1522static int timekeeping_inject_offset(const struct timespec64 *ts)
1523{
1524	int ret;
1525
1526	scoped_guard (raw_spinlock_irqsave, &tk_core.lock)
1527		ret = __timekeeping_inject_offset(&tk_core, ts);
1528
1529	/* Signal hrtimers about time change */
1530	if (!ret)
1531		clock_was_set(CLOCK_SET_WALL);
1532	return ret;
1533}
1534
1535/*
1536 * Indicates if there is an offset between the system clock and the hardware
1537 * clock/persistent clock/rtc.
1538 */
1539int persistent_clock_is_local;
1540
1541/*
1542 * Adjust the time obtained from the CMOS to be UTC time instead of
1543 * local time.
1544 *
1545 * This is ugly, but preferable to the alternatives.  Otherwise we
1546 * would either need to write a program to do it in /etc/rc (and risk
1547 * confusion if the program gets run more than once; it would also be
1548 * hard to make the program warp the clock precisely n hours)  or
1549 * compile in the timezone information into the kernel.  Bad, bad....
1550 *
1551 *						- TYT, 1992-01-01
1552 *
1553 * The best thing to do is to keep the CMOS clock in universal time (UTC)
1554 * as real UNIX machines always do it. This avoids all headaches about
1555 * daylight saving times and warping kernel clocks.
1556 */
1557void timekeeping_warp_clock(void)
1558{
1559	if (sys_tz.tz_minuteswest != 0) {
1560		struct timespec64 adjust;
1561
1562		persistent_clock_is_local = 1;
1563		adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1564		adjust.tv_nsec = 0;
1565		timekeeping_inject_offset(&adjust);
1566	}
1567}
1568
1569/*
1570 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1571 */
1572static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
1573{
1574	tk->tai_offset = tai_offset;
1575	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
1576}
1577
1578/*
1579 * change_clocksource - Swaps clocksources if a new one is available
1580 *
1581 * Accumulates current time interval and initializes new clocksource
1582 */
1583static int change_clocksource(void *data)
1584{
1585	struct clocksource *new = data, *old = NULL;
1586
1587	/*
1588	 * If the clocksource is in a module, get a module reference.
1589	 * Succeeds for built-in code (owner == NULL) as well. Abort if the
1590	 * reference can't be acquired.
1591	 */
1592	if (!try_module_get(new->owner))
1593		return 0;
1594
1595	/* Abort if the device can't be enabled */
1596	if (new->enable && new->enable(new) != 0) {
1597		module_put(new->owner);
1598		return 0;
1599	}
1600
1601	scoped_guard (raw_spinlock_irqsave, &tk_core.lock) {
1602		struct timekeeper *tks = &tk_core.shadow_timekeeper;
1603
1604		timekeeping_forward_now(tks);
1605		old = tks->tkr_mono.clock;
1606		tk_setup_internals(tks, new);
1607		timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
1608	}
1609
1610	tk_aux_update_clocksource();
1611
1612	if (old) {
1613		if (old->disable)
1614			old->disable(old);
1615		module_put(old->owner);
1616	}
1617
1618	return 0;
1619}
1620
1621/**
1622 * timekeeping_notify - Install a new clock source
1623 * @clock:		pointer to the clock source
1624 *
1625 * This function is called from clocksource.c after a new, better clock
1626 * source has been registered. The caller holds the clocksource_mutex.
1627 */
1628int timekeeping_notify(struct clocksource *clock)
1629{
1630	struct timekeeper *tk = &tk_core.timekeeper;
1631
1632	if (tk->tkr_mono.clock == clock)
1633		return 0;
1634	stop_machine(change_clocksource, clock, NULL);
1635	tick_clock_notify();
1636	return tk->tkr_mono.clock == clock ? 0 : -1;
1637}
1638
1639/**
1640 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1641 * @ts:		pointer to the timespec64 to be set
1642 *
1643 * Returns the raw monotonic time (completely un-modified by ntp)
1644 */
1645void ktime_get_raw_ts64(struct timespec64 *ts)
1646{
1647	struct timekeeper *tk = &tk_core.timekeeper;
1648	unsigned int seq;
1649	u64 nsecs;
1650
1651	do {
1652		seq = read_seqcount_begin(&tk_core.seq);
1653		ts->tv_sec = tk->raw_sec;
1654		nsecs = timekeeping_get_ns(&tk->tkr_raw);
1655
1656	} while (read_seqcount_retry(&tk_core.seq, seq));
1657
1658	ts->tv_nsec = 0;
1659	timespec64_add_ns(ts, nsecs);
1660}
1661EXPORT_SYMBOL(ktime_get_raw_ts64);
1662
1663/**
1664 * ktime_get_clock_ts64 - Returns time of a clock in a timespec
1665 * @id:		POSIX clock ID of the clock to read
1666 * @ts:		Pointer to the timespec64 to be set
1667 *
1668 * The timestamp is invalidated (@ts->sec is set to -1) if the
1669 * clock @id is not available.
1670 */
1671void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts)
1672{
1673	/* Invalidate time stamp */
1674	ts->tv_sec = -1;
1675	ts->tv_nsec = 0;
1676
1677	switch (id) {
1678	case CLOCK_REALTIME:
1679		ktime_get_real_ts64(ts);
1680		return;
1681	case CLOCK_MONOTONIC:
1682		ktime_get_ts64(ts);
1683		return;
1684	case CLOCK_MONOTONIC_RAW:
1685		ktime_get_raw_ts64(ts);
1686		return;
1687	case CLOCK_AUX ... CLOCK_AUX_LAST:
1688		if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
1689			ktime_get_aux_ts64(id, ts);
1690		return;
1691	default:
1692		WARN_ON_ONCE(1);
1693	}
1694}
1695EXPORT_SYMBOL_GPL(ktime_get_clock_ts64);
1696
1697/**
1698 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1699 */
1700int timekeeping_valid_for_hres(void)
1701{
1702	struct timekeeper *tk = &tk_core.timekeeper;
1703	unsigned int seq;
1704	int ret;
1705
1706	do {
1707		seq = read_seqcount_begin(&tk_core.seq);
1708
1709		ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
1710
1711	} while (read_seqcount_retry(&tk_core.seq, seq));
1712
1713	return ret;
1714}
1715
1716/**
1717 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1718 */
1719u64 timekeeping_max_deferment(void)
1720{
1721	struct timekeeper *tk = &tk_core.timekeeper;
1722	unsigned int seq;
1723	u64 ret;
1724
1725	do {
1726		seq = read_seqcount_begin(&tk_core.seq);
1727
1728		ret = tk->tkr_mono.clock->max_idle_ns;
1729
1730	} while (read_seqcount_retry(&tk_core.seq, seq));
1731
1732	return ret;
1733}
1734
1735/**
1736 * read_persistent_clock64 -  Return time from the persistent clock.
1737 * @ts: Pointer to the storage for the readout value
1738 *
1739 * Weak dummy function for arches that do not yet support it.
1740 * Reads the time from the battery backed persistent clock.
1741 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
1742 *
1743 *  XXX - Do be sure to remove it once all arches implement it.
1744 */
1745void __weak read_persistent_clock64(struct timespec64 *ts)
1746{
1747	ts->tv_sec = 0;
1748	ts->tv_nsec = 0;
1749}
1750
1751/**
1752 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1753 *                                        from the boot.
1754 * @wall_time:	  current time as returned by persistent clock
1755 * @boot_offset:  offset that is defined as wall_time - boot_time
1756 *
1757 * Weak dummy function for arches that do not yet support it.
1758 *
1759 * The default function calculates offset based on the current value of
1760 * local_clock(). This way architectures that support sched_clock() but don't
1761 * support dedicated boot time clock will provide the best estimate of the
1762 * boot time.
1763 */
1764void __weak __init
1765read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1766				     struct timespec64 *boot_offset)
1767{
1768	read_persistent_clock64(wall_time);
1769	*boot_offset = ns_to_timespec64(local_clock());
1770}
1771
1772static __init void tkd_basic_setup(struct tk_data *tkd, enum timekeeper_ids tk_id, bool valid)
1773{
1774	raw_spin_lock_init(&tkd->lock);
1775	seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);
1776	tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id;
1777	tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid;
1778}
1779
1780/*
1781 * Flag reflecting whether timekeeping_resume() has injected sleeptime.
1782 *
1783 * The flag starts of false and is only set when a suspend reaches
1784 * timekeeping_suspend(), timekeeping_resume() sets it to false when the
1785 * timekeeper clocksource is not stopping across suspend and has been
1786 * used to update sleep time. If the timekeeper clocksource has stopped
1787 * then the flag stays true and is used by the RTC resume code to decide
1788 * whether sleeptime must be injected and if so the flag gets false then.
1789 *
1790 * If a suspend fails before reaching timekeeping_resume() then the flag
1791 * stays false and prevents erroneous sleeptime injection.
1792 */
1793static bool suspend_timing_needed;
1794
1795/* Flag for if there is a persistent clock on this platform */
1796static bool persistent_clock_exists;
1797
1798/*
1799 * timekeeping_init - Initializes the clocksource and common timekeeping values
1800 */
1801void __init timekeeping_init(void)
1802{
1803	struct timespec64 wall_time, boot_offset, wall_to_mono;
1804	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1805	struct clocksource *clock;
1806
1807	tkd_basic_setup(&tk_core, TIMEKEEPER_CORE, true);
1808	tk_aux_setup();
1809
1810	read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
1811	if (timespec64_valid_settod(&wall_time) &&
1812	    timespec64_to_ns(&wall_time) > 0) {
1813		persistent_clock_exists = true;
1814	} else if (timespec64_to_ns(&wall_time) != 0) {
1815		pr_warn("Persistent clock returned invalid value");
1816		wall_time = (struct timespec64){0};
1817	}
1818
1819	if (timespec64_compare(&wall_time, &boot_offset) < 0)
1820		boot_offset = (struct timespec64){0};
1821
1822	/*
1823	 * We want set wall_to_mono, so the following is true:
1824	 * wall time + wall_to_mono = boot time
1825	 */
1826	wall_to_mono = timespec64_sub(boot_offset, wall_time);
1827
1828	guard(raw_spinlock_irqsave)(&tk_core.lock);
1829
1830	ntp_init();
1831
1832	clock = clocksource_default_clock();
1833	if (clock->enable)
1834		clock->enable(clock);
1835	tk_setup_internals(tks, clock);
1836
1837	tk_set_xtime(tks, &wall_time);
1838	tks->raw_sec = 0;
1839
1840	tk_set_wall_to_mono(tks, wall_to_mono);
1841
1842	timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
1843}
1844
1845/* time in seconds when suspend began for persistent clock */
1846static struct timespec64 timekeeping_suspend_time;
1847
1848/**
1849 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1850 * @tk:		Pointer to the timekeeper to be updated
1851 * @delta:	Pointer to the delta value in timespec64 format
1852 *
1853 * Takes a timespec offset measuring a suspend interval and properly
1854 * adds the sleep offset to the timekeeping variables.
1855 */
1856static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
1857					   const struct timespec64 *delta)
1858{
1859	if (!timespec64_valid_strict(delta)) {
1860		printk_deferred(KERN_WARNING
1861				"__timekeeping_inject_sleeptime: Invalid "
1862				"sleep delta value!\n");
1863		return;
1864	}
1865	tk_xtime_add(tk, delta);
1866	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
1867	tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
1868	tk_debug_account_sleep_time(delta);
1869}
1870
1871#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
1872/*
1873 * We have three kinds of time sources to use for sleep time
1874 * injection, the preference order is:
1875 * 1) non-stop clocksource
1876 * 2) persistent clock (ie: RTC accessible when irqs are off)
1877 * 3) RTC
1878 *
1879 * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1880 * If system has neither 1) nor 2), 3) will be used finally.
1881 *
1882 *
1883 * If timekeeping has injected sleeptime via either 1) or 2),
1884 * 3) becomes needless, so in this case we don't need to call
1885 * rtc_resume(), and this is what timekeeping_rtc_skipresume()
1886 * means.
1887 */
1888bool timekeeping_rtc_skipresume(void)
1889{
1890	return !suspend_timing_needed;
1891}
1892
1893/*
1894 * 1) can be determined whether to use or not only when doing
1895 * timekeeping_resume() which is invoked after rtc_suspend(),
1896 * so we can't skip rtc_suspend() surely if system has 1).
1897 *
1898 * But if system has 2), 2) will definitely be used, so in this
1899 * case we don't need to call rtc_suspend(), and this is what
1900 * timekeeping_rtc_skipsuspend() means.
1901 */
1902bool timekeeping_rtc_skipsuspend(void)
1903{
1904	return persistent_clock_exists;
1905}
1906
1907/**
1908 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1909 * @delta: pointer to a timespec64 delta value
1910 *
1911 * This hook is for architectures that cannot support read_persistent_clock64
1912 * because their RTC/persistent clock is only accessible when irqs are enabled.
1913 * and also don't have an effective nonstop clocksource.
1914 *
1915 * This function should only be called by rtc_resume(), and allows
1916 * a suspend offset to be injected into the timekeeping values.
1917 */
1918void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
1919{
1920	scoped_guard(raw_spinlock_irqsave, &tk_core.lock) {
1921		struct timekeeper *tks = &tk_core.shadow_timekeeper;
1922
1923		suspend_timing_needed = false;
1924		timekeeping_forward_now(tks);
1925		__timekeeping_inject_sleeptime(tks, delta);
1926		timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL);
1927	}
1928
1929	/* Signal hrtimers about time change */
1930	clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
1931}
1932#endif
1933
1934/**
1935 * timekeeping_resume - Resumes the generic timekeeping subsystem.
1936 */
1937void timekeeping_resume(void)
1938{
1939	struct timekeeper *tks = &tk_core.shadow_timekeeper;
1940	struct clocksource *clock = tks->tkr_mono.clock;
1941	struct timespec64 ts_new, ts_delta;
1942	bool inject_sleeptime = false;
1943	u64 cycle_now, nsec;
1944	unsigned long flags;
1945
1946	read_persistent_clock64(&ts_new);
1947
1948	clockevents_resume();
1949	clocksource_resume();
1950
1951	raw_spin_lock_irqsave(&tk_core.lock, flags);
1952
1953	/*
1954	 * After system resumes, we need to calculate the suspended time and
1955	 * compensate it for the OS time. There are 3 sources that could be
1956	 * used: Nonstop clocksource during suspend, persistent clock and rtc
1957	 * device.
1958	 *
1959	 * One specific platform may have 1 or 2 or all of them, and the
1960	 * preference will be:
1961	 *	suspend-nonstop clocksource -> persistent clock -> rtc
1962	 * The less preferred source will only be tried if there is no better
1963	 * usable source. The rtc part is handled separately in rtc core code.
1964	 */
1965	cycle_now = tk_clock_read(&tks->tkr_mono);
1966	nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1967	if (nsec > 0) {
1968		ts_delta = ns_to_timespec64(nsec);
1969		inject_sleeptime = true;
1970	} else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1971		ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
1972		inject_sleeptime = true;
1973	}
1974
1975	if (inject_sleeptime) {
1976		suspend_timing_needed = false;
1977		__timekeeping_inject_sleeptime(tks, &ts_delta);
1978	}
1979
1980	/* Re-base the last cycle value */
1981	tks->tkr_mono.cycle_last = cycle_now;
1982	tks->tkr_raw.cycle_last  = cycle_now;
1983
1984	tks->ntp_error = 0;
1985	timekeeping_suspended = 0;
1986	timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET);
1987	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
1988
1989	touch_softlockup_watchdog();
1990
1991	/* Resume the clockevent device(s) and hrtimers */
1992	tick_resume();
1993	/* Notify timerfd as resume is equivalent to clock_was_set() */
1994	timerfd_resume();
1995}
1996
1997static void timekeeping_syscore_resume(void *data)
1998{
1999	timekeeping_resume();
2000}
2001
2002int timekeeping_suspend(void)
2003{
2004	struct timekeeper *tks = &tk_core.shadow_timekeeper;
2005	struct timespec64 delta, delta_delta;
2006	static struct timespec64 old_delta;
2007	struct clocksource *curr_clock;
2008	unsigned long flags;
2009	u64 cycle_now;
2010
2011	read_persistent_clock64(&timekeeping_suspend_time);
2012
2013	/*
2014	 * On some systems the persistent_clock can not be detected at
2015	 * timekeeping_init by its return value, so if we see a valid
2016	 * value returned, update the persistent_clock_exists flag.
2017	 */
2018	if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
2019		persistent_clock_exists = true;
2020
2021	suspend_timing_needed = true;
2022
2023	raw_spin_lock_irqsave(&tk_core.lock, flags);
2024	timekeeping_forward_now(tks);
2025	timekeeping_suspended = 1;
2026
2027	/*
2028	 * Since we've called forward_now, cycle_last stores the value
2029	 * just read from the current clocksource. Save this to potentially
2030	 * use in suspend timing.
2031	 */
2032	curr_clock = tks->tkr_mono.clock;
2033	cycle_now = tks->tkr_mono.cycle_last;
2034	clocksource_start_suspend_timing(curr_clock, cycle_now);
2035
2036	if (persistent_clock_exists) {
2037		/*
2038		 * To avoid drift caused by repeated suspend/resumes,
2039		 * which each can add ~1 second drift error,
2040		 * try to compensate so the difference in system time
2041		 * and persistent_clock time stays close to constant.
2042		 */
2043		delta = timespec64_sub(tk_xtime(tks), timekeeping_suspend_time);
2044		delta_delta = timespec64_sub(delta, old_delta);
2045		if (abs(delta_delta.tv_sec) >= 2) {
2046			/*
2047			 * if delta_delta is too large, assume time correction
2048			 * has occurred and set old_delta to the current delta.
2049			 */
2050			old_delta = delta;
2051		} else {
2052			/* Otherwise try to adjust old_system to compensate */
2053			timekeeping_suspend_time =
2054				timespec64_add(timekeeping_suspend_time, delta_delta);
2055		}
2056	}
2057
2058	timekeeping_update_from_shadow(&tk_core, 0);
2059	halt_fast_timekeeper(tks);
2060	raw_spin_unlock_irqrestore(&tk_core.lock, flags);
2061
2062	tick_suspend();
2063	clocksource_suspend();
2064	clockevents_suspend();
2065
2066	return 0;
2067}
2068
2069static int timekeeping_syscore_suspend(void *data)
2070{
2071	return timekeeping_suspend();
2072}
2073
2074/* sysfs resume/suspend bits for timekeeping */
2075static const struct syscore_ops timekeeping_syscore_ops = {
2076	.resume		= timekeeping_syscore_resume,
2077	.suspend	= timekeeping_syscore_suspend,
2078};
2079
2080static struct syscore timekeeping_syscore = {
2081	.ops = &timekeeping_syscore_ops,
2082};
2083
2084static int __init timekeeping_init_ops(void)
2085{
2086	register_syscore(&timekeeping_syscore);
2087	return 0;
2088}
2089device_initcall(timekeeping_init_ops);
2090
2091/*
2092 * Apply a multiplier adjustment to the timekeeper
2093 */
2094static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
2095							 s64 offset,
2096							 s32 mult_adj)
2097{
2098	s64 interval = tk->cycle_interval;
2099
2100	if (mult_adj == 0) {
2101		return;
2102	} else if (mult_adj == -1) {
2103		interval = -interval;
2104		offset = -offset;
2105	} else if (mult_adj != 1) {
2106		interval *= mult_adj;
2107		offset *= mult_adj;
2108	}
2109
2110	/*
2111	 * So the following can be confusing.
2112	 *
2113	 * To keep things simple, lets assume mult_adj == 1 for now.
2114	 *
2115	 * When mult_adj != 1, remember that the interval and offset values
2116	 * have been appropriately scaled so the math is the same.
2117	 *
2118	 * The basic idea here is that we're increasing the multiplier
2119	 * by one, this causes the xtime_interval to be incremented by
2120	 * one cycle_interval. This is because:
2121	 *	xtime_interval = cycle_interval * mult
2122	 * So if mult is being incremented by one:
2123	 *	xtime_interval = cycle_interval * (mult + 1)
2124	 * Its the same as:
2125	 *	xtime_interval = (cycle_interval * mult) + cycle_interval
2126	 * Which can be shortened to:
2127	 *	xtime_interval += cycle_interval
2128	 *
2129	 * So offset stores the non-accumulated cycles. Thus the current
2130	 * time (in shifted nanoseconds) is:
2131	 *	now = (offset * adj) + xtime_nsec
2132	 * Now, even though we're adjusting the clock frequency, we have
2133	 * to keep time consistent. In other words, we can't jump back
2134	 * in time, and we also want to avoid jumping forward in time.
2135	 *
2136	 * So given the same offset value, we need the time to be the same
2137	 * both before and after the freq adjustment.
2138	 *	now = (offset * adj_1) + xtime_nsec_1
2139	 *	now = (offset * adj_2) + xtime_nsec_2
2140	 * So:
2141	 *	(offset * adj_1) + xtime_nsec_1 =
2142	 *		(offset * adj_2) + xtime_nsec_2
2143	 * And we know:
2144	 *	adj_2 = adj_1 + 1
2145	 * So:
2146	 *	(offset * adj_1) + xtime_nsec_1 =
2147	 *		(offset * (adj_1+1)) + xtime_nsec_2
2148	 *	(offset * adj_1) + xtime_nsec_1 =
2149	 *		(offset * adj_1) + offset + xtime_nsec_2
2150	 * Canceling the sides:
2151	 *	xtime_nsec_1 = offset + xtime_nsec_2
2152	 * Which gives us:
2153	 *	xtime_nsec_2 = xtime_nsec_1 - offset
2154	 * Which simplifies to:
2155	 *	xtime_nsec -= offset
2156	 */
2157	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
2158		/* NTP adjustment caused clocksource mult overflow */
2159		WARN_ON_ONCE(1);
2160		return;
2161	}
2162
2163	tk->tkr_mono.mult += mult_adj;
2164	tk->xtime_interval += interval;
2165	tk->tkr_mono.xtime_nsec -= offset;
2166}
2167
2168/*
2169 * Adjust the timekeeper's multiplier to the correct frequency
2170 * and also to reduce the accumulated error value.
2171 */
2172static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
2173{
2174	u64 ntp_tl = ntp_tick_length(tk->id);
2175	u32 mult;
2176
2177	/*
2178	 * Determine the multiplier from the current NTP tick length.
2179	 * Avoid expensive division when the tick length doesn't change.
2180	 */
2181	if (likely(tk->ntp_tick == ntp_tl)) {
2182		mult = tk->tkr_mono.mult - tk->ntp_err_mult;
2183	} else {
2184		tk->ntp_tick = ntp_tl;
2185		mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
2186				 tk->xtime_remainder, tk->cycle_interval);
2187	}
2188
2189	/*
2190	 * If the clock is behind the NTP time, increase the multiplier by 1
2191	 * to catch up with it. If it's ahead and there was a remainder in the
2192	 * tick division, the clock will slow down. Otherwise it will stay
2193	 * ahead until the tick length changes to a non-divisible value.
2194	 */
2195	tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
2196	mult += tk->ntp_err_mult;
2197
2198	timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
2199
2200	if (unlikely(tk->tkr_mono.clock->maxadj &&
2201		(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
2202			> tk->tkr_mono.clock->maxadj))) {
2203		printk_once(KERN_WARNING
2204			"Adjusting %s more than 11%% (%ld vs %ld)\n",
2205			tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
2206			(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
2207	}
2208
2209	/*
2210	 * It may be possible that when we entered this function, xtime_nsec
2211	 * was very small.  Further, if we're slightly speeding the clocksource
2212	 * in the code above, its possible the required corrective factor to
2213	 * xtime_nsec could cause it to underflow.
2214	 *
2215	 * Now, since we have already accumulated the second and the NTP
2216	 * subsystem has been notified via second_overflow(), we need to skip
2217	 * the next update.
2218	 */
2219	if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
2220		tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
2221							tk->tkr_mono.shift;
2222		tk->xtime_sec--;
2223		tk->skip_second_overflow = 1;
2224	}
2225}
2226
2227/*
2228 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2229 *
2230 * Helper function that accumulates the nsecs greater than a second
2231 * from the xtime_nsec field to the xtime_secs field.
2232 * It also calls into the NTP code to handle leapsecond processing.
2233 */
2234static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
2235{
2236	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
2237	unsigned int clock_set = 0;
2238
2239	while (tk->tkr_mono.xtime_nsec >= nsecps) {
2240		int leap;
2241
2242		tk->tkr_mono.xtime_nsec -= nsecps;
2243		tk->xtime_sec++;
2244
2245		/*
2246		 * Skip NTP update if this second was accumulated before,
2247		 * i.e. xtime_nsec underflowed in timekeeping_adjust()
2248		 */
2249		if (unlikely(tk->skip_second_overflow)) {
2250			tk->skip_second_overflow = 0;
2251			continue;
2252		}
2253
2254		/* Figure out if its a leap sec and apply if needed */
2255		leap = second_overflow(tk->id, tk->xtime_sec);
2256		if (unlikely(leap)) {
2257			struct timespec64 ts;
2258
2259			tk->xtime_sec += leap;
2260
2261			ts.tv_sec = leap;
2262			ts.tv_nsec = 0;
2263			tk_set_wall_to_mono(tk,
2264				timespec64_sub(tk->wall_to_monotonic, ts));
2265
2266			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
2267
2268			clock_set = TK_CLOCK_WAS_SET;
2269		}
2270	}
2271	return clock_set;
2272}
2273
2274/*
2275 * logarithmic_accumulation - shifted accumulation of cycles
2276 *
2277 * This functions accumulates a shifted interval of cycles into
2278 * a shifted interval nanoseconds. Allows for O(log) accumulation
2279 * loop.
2280 *
2281 * Returns the unconsumed cycles.
2282 */
2283static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2284				    u32 shift, unsigned int *clock_set)
2285{
2286	u64 interval = tk->cycle_interval << shift;
2287	u64 snsec_per_sec;
2288
2289	/* If the offset is smaller than a shifted interval, do nothing */
2290	if (offset < interval)
2291		return offset;
2292
2293	/* Accumulate one shifted interval */
2294	offset -= interval;
2295	tk->tkr_mono.cycle_last += interval;
2296	tk->tkr_raw.cycle_last  += interval;
2297
2298	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
2299	*clock_set |= accumulate_nsecs_to_secs(tk);
2300
2301	/* Accumulate raw time */
2302	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2303	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2304	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2305		tk->tkr_raw.xtime_nsec -= snsec_per_sec;
2306		tk->raw_sec++;
2307	}
2308
2309	/* Accumulate error between NTP and clock interval */
2310	tk->ntp_error += tk->ntp_tick << shift;
2311	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2312						(tk->ntp_error_shift + shift);
2313
2314	return offset;
2315}
2316
2317/*
2318 * timekeeping_advance - Updates the timekeeper to the current time and
2319 * current NTP tick length
2320 */
2321static bool __timekeeping_advance(struct tk_data *tkd, enum timekeeping_adv_mode mode)
2322{
2323	struct timekeeper *tk = &tkd->shadow_timekeeper;
2324	struct timekeeper *real_tk = &tkd->timekeeper;
2325	unsigned int clock_set = 0;
2326	int shift = 0, maxshift;
2327	u64 offset, orig_offset;
2328
2329	/* Make sure we're fully resumed: */
2330	if (unlikely(timekeeping_suspended))
2331		return false;
2332
2333	offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
2334				   tk->tkr_mono.cycle_last, tk->tkr_mono.mask,
2335				   tk->tkr_mono.clock->max_raw_delta);
2336	orig_offset = offset;
2337	/* Check if there's really nothing to do */
2338	if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
2339		return false;
2340
2341	/*
2342	 * With NO_HZ we may have to accumulate many cycle_intervals
2343	 * (think "ticks") worth of time at once. To do this efficiently,
2344	 * we calculate the largest doubling multiple of cycle_intervals
2345	 * that is smaller than the offset.  We then accumulate that
2346	 * chunk in one go, and then try to consume the next smaller
2347	 * doubled multiple.
2348	 */
2349	shift = ilog2(offset) - ilog2(tk->cycle_interval);
2350	shift = max(0, shift);
2351	/* Bound shift to one less than what overflows tick_length */
2352	maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1;
2353	shift = min(shift, maxshift);
2354	while (offset >= tk->cycle_interval) {
2355		offset = logarithmic_accumulation(tk, offset, shift, &clock_set);
2356		if (offset < tk->cycle_interval<<shift)
2357			shift--;
2358	}
2359
2360	/* Adjust the multiplier to correct NTP error */
2361	timekeeping_adjust(tk, offset);
2362
2363	/*
2364	 * Finally, make sure that after the rounding
2365	 * xtime_nsec isn't larger than NSEC_PER_SEC
2366	 */
2367	clock_set |= accumulate_nsecs_to_secs(tk);
2368
2369	/*
2370	 * To avoid inconsistencies caused adjtimex TK_ADV_FREQ calls
2371	 * making small negative adjustments to the base xtime_nsec
2372	 * value, only update the coarse clocks if we accumulated time
2373	 */
2374	if (orig_offset != offset)
2375		tk_update_coarse_nsecs(tk);
2376
2377	timekeeping_update_from_shadow(tkd, clock_set);
2378
2379	return !!clock_set;
2380}
2381
2382static bool timekeeping_advance(enum timekeeping_adv_mode mode)
2383{
2384	guard(raw_spinlock_irqsave)(&tk_core.lock);
2385	return __timekeeping_advance(&tk_core, mode);
2386}
2387
2388/**
2389 * update_wall_time - Uses the current clocksource to increment the wall time
2390 *
2391 * It also updates the enabled auxiliary clock timekeepers
2392 */
2393void update_wall_time(void)
2394{
2395	if (timekeeping_advance(TK_ADV_TICK))
2396		clock_was_set_delayed();
2397	tk_aux_advance();
2398}
2399
2400/**
2401 * getboottime64 - Return the real time of system boot.
2402 * @ts:		pointer to the timespec64 to be set
2403 *
2404 * Returns the wall-time of boot in a timespec64.
2405 *
2406 * This is based on the wall_to_monotonic offset and the total suspend
2407 * time. Calls to settimeofday will affect the value returned (which
2408 * basically means that however wrong your real time clock is at boot time,
2409 * you get the right time here).
2410 */
2411void getboottime64(struct timespec64 *ts)
2412{
2413	struct timekeeper *tk = &tk_core.timekeeper;
2414	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
2415
2416	*ts = ktime_to_timespec64(t);
2417}
2418EXPORT_SYMBOL_GPL(getboottime64);
2419
2420void ktime_get_coarse_real_ts64(struct timespec64 *ts)
2421{
2422	struct timekeeper *tk = &tk_core.timekeeper;
2423	unsigned int seq;
2424
2425	do {
2426		seq = read_seqcount_begin(&tk_core.seq);
2427
2428		*ts = tk_xtime_coarse(tk);
2429	} while (read_seqcount_retry(&tk_core.seq, seq));
2430}
2431EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
2432
2433/**
2434 * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
2435 * @ts:		timespec64 to be filled
2436 *
2437 * Fetch the global mg_floor value, convert it to realtime and compare it
2438 * to the current coarse-grained time. Fill @ts with whichever is
2439 * latest. Note that this is a filesystem-specific interface and should be
2440 * avoided outside of that context.
2441 */
2442void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts)
2443{
2444	struct timekeeper *tk = &tk_core.timekeeper;
2445	u64 floor = atomic64_read(&mg_floor);
2446	ktime_t f_real, offset, coarse;
2447	unsigned int seq;
2448
2449	do {
2450		seq = read_seqcount_begin(&tk_core.seq);
2451		*ts = tk_xtime_coarse(tk);
2452		offset = tk_core.timekeeper.offs_real;
2453	} while (read_seqcount_retry(&tk_core.seq, seq));
2454
2455	coarse = timespec64_to_ktime(*ts);
2456	f_real = ktime_add(floor, offset);
2457	if (ktime_after(f_real, coarse))
2458		*ts = ktime_to_timespec64(f_real);
2459}
2460
2461/**
2462 * ktime_get_real_ts64_mg - attempt to update floor value and return result
2463 * @ts:		pointer to the timespec to be set
2464 *
2465 * Get a monotonic fine-grained time value and attempt to swap it into
2466 * mg_floor. If that succeeds then accept the new floor value. If it fails
2467 * then another task raced in during the interim time and updated the
2468 * floor.  Since any update to the floor must be later than the previous
2469 * floor, either outcome is acceptable.
2470 *
2471 * Typically this will be called after calling ktime_get_coarse_real_ts64_mg(),
2472 * and determining that the resulting coarse-grained timestamp did not effect
2473 * a change in ctime. Any more recent floor value would effect a change to
2474 * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
2475 *
2476 * @ts will be filled with the latest floor value, regardless of the outcome of
2477 * the cmpxchg. Note that this is a filesystem specific interface and should be
2478 * avoided outside of that context.
2479 */
2480void ktime_get_real_ts64_mg(struct timespec64 *ts)
2481{
2482	struct timekeeper *tk = &tk_core.timekeeper;
2483	ktime_t old = atomic64_read(&mg_floor);
2484	ktime_t offset, mono;
2485	unsigned int seq;
2486	u64 nsecs;
2487
2488	do {
2489		seq = read_seqcount_begin(&tk_core.seq);
2490
2491		ts->tv_sec = tk->xtime_sec;
2492		mono = tk->tkr_mono.base;
2493		nsecs = timekeeping_get_ns(&tk->tkr_mono);
2494		offset = tk_core.timekeeper.offs_real;
2495	} while (read_seqcount_retry(&tk_core.seq, seq));
2496
2497	mono = ktime_add_ns(mono, nsecs);
2498
2499	/*
2500	 * Attempt to update the floor with the new time value. As any
2501	 * update must be later then the existing floor, and would effect
2502	 * a change to ctime from the perspective of the current task,
2503	 * accept the resulting floor value regardless of the outcome of
2504	 * the swap.
2505	 */
2506	if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) {
2507		ts->tv_nsec = 0;
2508		timespec64_add_ns(ts, nsecs);
2509		timekeeping_inc_mg_floor_swaps();
2510	} else {
2511		/*
2512		 * Another task changed mg_floor since "old" was fetched.
2513		 * "old" has been updated with the latest value of "mg_floor".
2514		 * That value is newer than the previous floor value, which
2515		 * is enough to effect a change to ctime. Accept it.
2516		 */
2517		*ts = ktime_to_timespec64(ktime_add(old, offset));
2518	}
2519}
2520
2521void ktime_get_coarse_ts64(struct timespec64 *ts)
2522{
2523	struct timekeeper *tk = &tk_core.timekeeper;
2524	struct timespec64 now, mono;
2525	unsigned int seq;
2526
2527	do {
2528		seq = read_seqcount_begin(&tk_core.seq);
2529
2530		now = tk_xtime_coarse(tk);
2531		mono = tk->wall_to_monotonic;
2532	} while (read_seqcount_retry(&tk_core.seq, seq));
2533
2534	set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
2535				  now.tv_nsec + mono.tv_nsec);
2536}
2537EXPORT_SYMBOL(ktime_get_coarse_ts64);
2538
2539/*
2540 * Must hold jiffies_lock
2541 */
2542void do_timer(unsigned long ticks)
2543{
2544	jiffies_64 += ticks;
2545	calc_global_load();
2546}
2547
2548/**
2549 * ktime_get_update_offsets_now - hrtimer helper
2550 * @cwsseq:	pointer to check and store the clock was set sequence number
2551 * @offs_real:	pointer to storage for monotonic -> realtime offset
2552 * @offs_boot:	pointer to storage for monotonic -> boottime offset
2553 * @offs_tai:	pointer to storage for monotonic -> clock tai offset
2554 *
2555 * Returns current monotonic time and updates the offsets if the
2556 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2557 * different.
2558 *
2559 * Called from hrtimer_interrupt() or retrigger_next_event()
2560 */
2561ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
2562				     ktime_t *offs_boot, ktime_t *offs_tai)
2563{
2564	struct timekeeper *tk = &tk_core.timekeeper;
2565	unsigned int seq;
2566	ktime_t base;
2567	u64 nsecs;
2568
2569	do {
2570		seq = read_seqcount_begin(&tk_core.seq);
2571
2572		base = tk->tkr_mono.base;
2573		nsecs = timekeeping_get_ns(&tk->tkr_mono);
2574		base = ktime_add_ns(base, nsecs);
2575
2576		if (*cwsseq != tk->clock_was_set_seq) {
2577			*cwsseq = tk->clock_was_set_seq;
2578			*offs_real = tk->offs_real;
2579			*offs_boot = tk->offs_boot;
2580			*offs_tai = tk->offs_tai;
2581		}
2582
2583		/* Handle leapsecond insertion adjustments */
2584		if (unlikely(base >= tk->next_leap_ktime))
2585			*offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2586
2587	} while (read_seqcount_retry(&tk_core.seq, seq));
2588
2589	return base;
2590}
2591
2592/*
2593 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2594 */
2595static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock)
2596{
2597	if (txc->modes & ADJ_ADJTIME) {
2598		/* singleshot must not be used with any other mode bits */
2599		if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2600			return -EINVAL;
2601		if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2602		    !capable(CAP_SYS_TIME))
2603			return -EPERM;
2604	} else {
2605		/* In order to modify anything, you gotta be super-user! */
2606		if (txc->modes && !capable(CAP_SYS_TIME))
2607			return -EPERM;
2608		/*
2609		 * if the quartz is off by more than 10% then
2610		 * something is VERY wrong!
2611		 */
2612		if (txc->modes & ADJ_TICK &&
2613		    (txc->tick <  900000/USER_HZ ||
2614		     txc->tick > 1100000/USER_HZ))
2615			return -EINVAL;
2616	}
2617
2618	if (txc->modes & ADJ_SETOFFSET) {
2619		/* In order to inject time, you gotta be super-user! */
2620		if (!capable(CAP_SYS_TIME))
2621			return -EPERM;
2622
2623		/*
2624		 * Validate if a timespec/timeval used to inject a time
2625		 * offset is valid.  Offsets can be positive or negative, so
2626		 * we don't check tv_sec. The value of the timeval/timespec
2627		 * is the sum of its fields,but *NOTE*:
2628		 * The field tv_usec/tv_nsec must always be non-negative and
2629		 * we can't have more nanoseconds/microseconds than a second.
2630		 */
2631		if (txc->time.tv_usec < 0)
2632			return -EINVAL;
2633
2634		if (txc->modes & ADJ_NANO) {
2635			if (txc->time.tv_usec >= NSEC_PER_SEC)
2636				return -EINVAL;
2637		} else {
2638			if (txc->time.tv_usec >= USEC_PER_SEC)
2639				return -EINVAL;
2640		}
2641	}
2642
2643	/*
2644	 * Check for potential multiplication overflows that can
2645	 * only happen on 64-bit systems:
2646	 */
2647	if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2648		if (LLONG_MIN / PPM_SCALE > txc->freq)
2649			return -EINVAL;
2650		if (LLONG_MAX / PPM_SCALE < txc->freq)
2651			return -EINVAL;
2652	}
2653
2654	if (aux_clock) {
2655		/* Auxiliary clocks are similar to TAI and do not have leap seconds */
2656		if (txc->modes & ADJ_STATUS &&
2657		    txc->status & (STA_INS | STA_DEL))
2658			return -EINVAL;
2659
2660		/* No TAI offset setting */
2661		if (txc->modes & ADJ_TAI)
2662			return -EINVAL;
2663
2664		/* No PPS support either */
2665		if (txc->modes & ADJ_STATUS &&
2666		    txc->status & (STA_PPSFREQ | STA_PPSTIME))
2667			return -EINVAL;
2668	}
2669
2670	return 0;
2671}
2672
2673/**
2674 * random_get_entropy_fallback - Returns the raw clock source value,
2675 * used by random.c for platforms with no valid random_get_entropy().
2676 */
2677unsigned long random_get_entropy_fallback(void)
2678{
2679	struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
2680	struct clocksource *clock = READ_ONCE(tkr->clock);
2681
2682	if (unlikely(timekeeping_suspended || !clock))
2683		return 0;
2684	return clock->read(clock);
2685}
2686EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
2687
2688struct adjtimex_result {
2689	struct audit_ntp_data	ad;
2690	struct timespec64	delta;
2691	bool			clock_set;
2692};
2693
2694static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc,
2695			 struct adjtimex_result *result)
2696{
2697	struct timekeeper *tks = &tkd->shadow_timekeeper;
2698	bool aux_clock = !timekeeper_is_core_tk(tks);
2699	struct timespec64 ts;
2700	s32 orig_tai, tai;
2701	int ret;
2702
2703	/* Validate the data before disabling interrupts */
2704	ret = timekeeping_validate_timex(txc, aux_clock);
2705	if (ret)
2706		return ret;
2707	add_device_randomness(txc, sizeof(*txc));
2708
2709	if (!aux_clock)
2710		ktime_get_real_ts64(&ts);
2711	else
2712		tk_get_aux_ts64(tkd->timekeeper.id, &ts);
2713
2714	add_device_randomness(&ts, sizeof(ts));
2715
2716	guard(raw_spinlock_irqsave)(&tkd->lock);
2717
2718	if (!tks->clock_valid)
2719		return -ENODEV;
2720
2721	if (txc->modes & ADJ_SETOFFSET) {
2722		result->delta.tv_sec  = txc->time.tv_sec;
2723		result->delta.tv_nsec = txc->time.tv_usec;
2724		if (!(txc->modes & ADJ_NANO))
2725			result->delta.tv_nsec *= 1000;
2726		ret = __timekeeping_inject_offset(tkd, &result->delta);
2727		if (ret)
2728			return ret;
2729		result->clock_set = true;
2730	}
2731
2732	orig_tai = tai = tks->tai_offset;
2733	ret = ntp_adjtimex(tks->id, txc, &ts, &tai, &result->ad);
2734
2735	if (tai != orig_tai) {
2736		__timekeeping_set_tai_offset(tks, tai);
2737		timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET);
2738		result->clock_set = true;
2739	} else {
2740		tk_update_leap_state_all(tkd);
2741	}
2742
2743	/* Update the multiplier immediately if frequency was set directly */
2744	if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
2745		result->clock_set |= __timekeeping_advance(tkd, TK_ADV_FREQ);
2746
2747	return ret;
2748}
2749
2750/**
2751 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2752 * @txc:	Pointer to kernel_timex structure containing NTP parameters
2753 */
2754int do_adjtimex(struct __kernel_timex *txc)
2755{
2756	struct adjtimex_result result = { };
2757	int ret;
2758
2759	ret = __do_adjtimex(&tk_core, txc, &result);
2760	if (ret < 0)
2761		return ret;
2762
2763	if (txc->modes & ADJ_SETOFFSET)
2764		audit_tk_injoffset(result.delta);
2765
2766	audit_ntp_log(&result.ad);
2767
2768	if (result.clock_set)
2769		clock_was_set(CLOCK_SET_WALL);
2770
2771	ntp_notify_cmos_timer(result.delta.tv_sec != 0);
2772
2773	return ret;
2774}
2775
2776/*
2777 * Invoked from NTP with the time keeper lock held, so lockless access is
2778 * fine.
2779 */
2780long ktime_get_ntp_seconds(unsigned int id)
2781{
2782	return timekeeper_data[id].timekeeper.xtime_sec;
2783}
2784
2785#ifdef CONFIG_NTP_PPS
2786/**
2787 * hardpps() - Accessor function to NTP __hardpps function
2788 * @phase_ts:	Pointer to timespec64 structure representing phase timestamp
2789 * @raw_ts:	Pointer to timespec64 structure representing raw timestamp
2790 */
2791void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
2792{
2793	guard(raw_spinlock_irqsave)(&tk_core.lock);
2794	__hardpps(phase_ts, raw_ts);
2795}
2796EXPORT_SYMBOL(hardpps);
2797#endif /* CONFIG_NTP_PPS */
2798
2799#ifdef CONFIG_POSIX_AUX_CLOCKS
2800#include "posix-timers.h"
2801
2802/*
2803 * Bitmap for the activated auxiliary timekeepers to allow lockless quick
2804 * checks in the hot paths without touching extra cache lines. If set, then
2805 * the state of the corresponding timekeeper has to be re-checked under
2806 * timekeeper::lock.
2807 */
2808static unsigned long aux_timekeepers;
2809
2810static inline unsigned int clockid_to_tkid(unsigned int id)
2811{
2812	return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX;
2813}
2814
2815static inline struct tk_data *aux_get_tk_data(clockid_t id)
2816{
2817	if (!clockid_aux_valid(id))
2818		return NULL;
2819	return &timekeeper_data[clockid_to_tkid(id)];
2820}
2821
2822/* Invoked from timekeeping after a clocksource change */
2823static void tk_aux_update_clocksource(void)
2824{
2825	unsigned long active = READ_ONCE(aux_timekeepers);
2826	unsigned int id;
2827
2828	for_each_set_bit(id, &active, BITS_PER_LONG) {
2829		struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
2830		struct timekeeper *tks = &tkd->shadow_timekeeper;
2831
2832		guard(raw_spinlock_irqsave)(&tkd->lock);
2833		if (!tks->clock_valid)
2834			continue;
2835
2836		timekeeping_forward_now(tks);
2837		tk_setup_internals(tks, tk_core.timekeeper.tkr_mono.clock);
2838		timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL);
2839	}
2840}
2841
2842static void tk_aux_advance(void)
2843{
2844	unsigned long active = READ_ONCE(aux_timekeepers);
2845	unsigned int id;
2846
2847	/* Lockless quick check to avoid extra cache lines */
2848	for_each_set_bit(id, &active, BITS_PER_LONG) {
2849		struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST];
2850
2851		guard(raw_spinlock)(&aux_tkd->lock);
2852		if (aux_tkd->shadow_timekeeper.clock_valid)
2853			__timekeeping_advance(aux_tkd, TK_ADV_TICK);
2854	}
2855}
2856
2857/**
2858 * ktime_get_aux - Get time for a AUX clock
2859 * @id:	ID of the clock to read (CLOCK_AUX...)
2860 * @kt:	Pointer to ktime_t to store the time stamp
2861 *
2862 * Returns: True if the timestamp is valid, false otherwise
2863 */
2864bool ktime_get_aux(clockid_t id, ktime_t *kt)
2865{
2866	struct tk_data *aux_tkd = aux_get_tk_data(id);
2867	struct timekeeper *aux_tk;
2868	unsigned int seq;
2869	ktime_t base;
2870	u64 nsecs;
2871
2872	WARN_ON(timekeeping_suspended);
2873
2874	if (!aux_tkd)
2875		return false;
2876
2877	aux_tk = &aux_tkd->timekeeper;
2878	do {
2879		seq = read_seqcount_begin(&aux_tkd->seq);
2880		if (!aux_tk->clock_valid)
2881			return false;
2882
2883		base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux);
2884		nsecs = timekeeping_get_ns(&aux_tk->tkr_mono);
2885	} while (read_seqcount_retry(&aux_tkd->seq, seq));
2886
2887	*kt = ktime_add_ns(base, nsecs);
2888	return true;
2889}
2890EXPORT_SYMBOL_GPL(ktime_get_aux);
2891
2892/**
2893 * ktime_get_aux_ts64 - Get time for a AUX clock
2894 * @id:	ID of the clock to read (CLOCK_AUX...)
2895 * @ts:	Pointer to timespec64 to store the time stamp
2896 *
2897 * Returns: True if the timestamp is valid, false otherwise
2898 */
2899bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts)
2900{
2901	ktime_t now;
2902
2903	if (!ktime_get_aux(id, &now))
2904		return false;
2905	*ts = ktime_to_timespec64(now);
2906	return true;
2907}
2908EXPORT_SYMBOL_GPL(ktime_get_aux_ts64);
2909
2910static int aux_get_res(clockid_t id, struct timespec64 *tp)
2911{
2912	if (!clockid_aux_valid(id))
2913		return -ENODEV;
2914
2915	tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC;
2916	tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC;
2917	return 0;
2918}
2919
2920static int aux_get_timespec(clockid_t id, struct timespec64 *tp)
2921{
2922	return ktime_get_aux_ts64(id, tp) ? 0 : -ENODEV;
2923}
2924
2925static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew)
2926{
2927	struct tk_data *aux_tkd = aux_get_tk_data(id);
2928	struct timekeeper *aux_tks;
2929	ktime_t tnow, nsecs;
2930
2931	if (!timespec64_valid_settod(tnew))
2932		return -EINVAL;
2933	if (!aux_tkd)
2934		return -ENODEV;
2935
2936	aux_tks = &aux_tkd->shadow_timekeeper;
2937
2938	guard(raw_spinlock_irq)(&aux_tkd->lock);
2939	if (!aux_tks->clock_valid)
2940		return -ENODEV;
2941
2942	/* Forward the timekeeper base time */
2943	timekeeping_forward_now(aux_tks);
2944	/*
2945	 * Get the updated base time. tkr_mono.base has not been
2946	 * updated yet, so do that first. That makes the update
2947	 * in timekeeping_update_from_shadow() redundant, but
2948	 * that's harmless. After that @tnow can be calculated
2949	 * by using tkr_mono::cycle_last, which has been set
2950	 * by timekeeping_forward_now().
2951	 */
2952	tk_update_ktime_data(aux_tks);
2953	nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last);
2954	tnow = ktime_add(aux_tks->tkr_mono.base, nsecs);
2955
2956	/*
2957	 * Calculate the new AUX offset as delta to @tnow ("monotonic").
2958	 * That avoids all the tk::xtime back and forth conversions as
2959	 * xtime ("realtime") is not applicable for auxiliary clocks and
2960	 * kept in sync with "monotonic".
2961	 */
2962	tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow));
2963
2964	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
2965	return 0;
2966}
2967
2968static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc)
2969{
2970	struct tk_data *aux_tkd = aux_get_tk_data(id);
2971	struct adjtimex_result result = { };
2972
2973	if (!aux_tkd)
2974		return -ENODEV;
2975
2976	/*
2977	 * @result is ignored for now as there are neither hrtimers nor a
2978	 * RTC related to auxiliary clocks for now.
2979	 */
2980	return __do_adjtimex(aux_tkd, txc, &result);
2981}
2982
2983const struct k_clock clock_aux = {
2984	.clock_getres		= aux_get_res,
2985	.clock_get_timespec	= aux_get_timespec,
2986	.clock_set		= aux_clock_set,
2987	.clock_adj		= aux_clock_adj,
2988};
2989
2990static void aux_clock_enable(clockid_t id)
2991{
2992	struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw;
2993	struct tk_data *aux_tkd = aux_get_tk_data(id);
2994	struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper;
2995
2996	/* Prevent the core timekeeper from changing. */
2997	guard(raw_spinlock_irq)(&tk_core.lock);
2998
2999	/*
3000	 * Setup the auxiliary clock assuming that the raw core timekeeper
3001	 * clock frequency conversion is close enough. Userspace has to
3002	 * adjust for the deviation via clock_adjtime(2).
3003	 */
3004	guard(raw_spinlock_nested)(&aux_tkd->lock);
3005
3006	/* Remove leftovers of a previous registration */
3007	memset(aux_tks, 0, sizeof(*aux_tks));
3008	/* Restore the timekeeper id */
3009	aux_tks->id = aux_tkd->timekeeper.id;
3010	/* Setup the timekeeper based on the current system clocksource */
3011	tk_setup_internals(aux_tks, tkr_raw->clock);
3012
3013	/* Mark it valid and set it live */
3014	aux_tks->clock_valid = true;
3015	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
3016}
3017
3018static void aux_clock_disable(clockid_t id)
3019{
3020	struct tk_data *aux_tkd = aux_get_tk_data(id);
3021
3022	guard(raw_spinlock_irq)(&aux_tkd->lock);
3023	aux_tkd->shadow_timekeeper.clock_valid = false;
3024	timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL);
3025}
3026
3027static DEFINE_MUTEX(aux_clock_mutex);
3028
3029static ssize_t aux_clock_enable_store(struct kobject *kobj, struct kobj_attribute *attr,
3030				      const char *buf, size_t count)
3031{
3032	/* Lazy atoi() as name is "0..7" */
3033	int id = kobj->name[0] & 0x7;
3034	bool enable;
3035
3036	if (!capable(CAP_SYS_TIME))
3037		return -EPERM;
3038
3039	if (kstrtobool(buf, &enable) < 0)
3040		return -EINVAL;
3041
3042	guard(mutex)(&aux_clock_mutex);
3043	if (enable == test_bit(id, &aux_timekeepers))
3044		return count;
3045
3046	if (enable) {
3047		aux_clock_enable(CLOCK_AUX + id);
3048		set_bit(id, &aux_timekeepers);
3049	} else {
3050		aux_clock_disable(CLOCK_AUX + id);
3051		clear_bit(id, &aux_timekeepers);
3052	}
3053	return count;
3054}
3055
3056static ssize_t aux_clock_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
3057{
3058	unsigned long active = READ_ONCE(aux_timekeepers);
3059	/* Lazy atoi() as name is "0..7" */
3060	int id = kobj->name[0] & 0x7;
3061
3062	return sysfs_emit(buf, "%d\n", test_bit(id, &active));
3063}
3064
3065static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable);
3066
3067static struct attribute *aux_clock_enable_attrs[] = {
3068	&aux_clock_enable_attr.attr,
3069	NULL
3070};
3071
3072static const struct attribute_group aux_clock_enable_attr_group = {
3073	.attrs = aux_clock_enable_attrs,
3074};
3075
3076static int __init tk_aux_sysfs_init(void)
3077{
3078	struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj);
3079	int ret = -ENOMEM;
3080
3081	if (!tko)
3082		return ret;
3083
3084	auxo = kobject_create_and_add("aux_clocks", tko);
3085	if (!auxo)
3086		goto err_clean;
3087
3088	for (int i = 0; i < MAX_AUX_CLOCKS; i++) {
3089		char id[2] = { [0] = '0' + i, };
3090		struct kobject *clk = kobject_create_and_add(id, auxo);
3091
3092		if (!clk) {
3093			ret = -ENOMEM;
3094			goto err_clean;
3095		}
3096
3097		ret = sysfs_create_group(clk, &aux_clock_enable_attr_group);
3098		if (ret)
3099			goto err_clean;
3100	}
3101	return 0;
3102
3103err_clean:
3104	kobject_put(auxo);
3105	kobject_put(tko);
3106	return ret;
3107}
3108late_initcall(tk_aux_sysfs_init);
3109
3110static __init void tk_aux_setup(void)
3111{
3112	for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++)
3113		tkd_basic_setup(&timekeeper_data[i], i, false);
3114}
3115#endif /* CONFIG_POSIX_AUX_CLOCKS */