vtime.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *    Virtual cpu timer based timer functions.
  4 *
  5 *    Copyright IBM Corp. 2004, 2012
  6 *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
  7 */
  8
  9#include <linux/kernel_stat.h>
 10#include <linux/export.h>
 11#include <linux/kernel.h>
 12#include <linux/timex.h>
 13#include <linux/types.h>
 14#include <linux/time.h>
 15#include <asm/alternative.h>
 16#include <asm/cputime.h>
 17#include <asm/vtimer.h>
 18#include <asm/vtime.h>
 19#include <asm/cpu_mf.h>
 20#include <asm/smp.h>
 21
 22#include "entry.h"
 23
 24static void virt_timer_expire(void);
 25
 26static LIST_HEAD(virt_timer_list);
 27static DEFINE_SPINLOCK(virt_timer_lock);
 28static atomic64_t virt_timer_current;
 29static atomic64_t virt_timer_elapsed;
 30
 31DEFINE_PER_CPU(u64, mt_cycles[8]);
 32static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
 33static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
 34static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
 35
 36static inline void set_vtimer(u64 expires)
 37{
 38	struct lowcore *lc = get_lowcore();
 39	u64 timer;
 40
 41	asm volatile(
 42		"	stpt	%0\n"	/* Store current cpu timer value */
 43		"	spt	%1"	/* Set new value imm. afterwards */
 44		: "=Q" (timer) : "Q" (expires));
 45	lc->system_timer += lc->last_update_timer - timer;
 46	lc->last_update_timer = expires;
 47}
 48
 49static inline int virt_timer_forward(u64 elapsed)
 50{
 51	BUG_ON(!irqs_disabled());
 52
 53	if (list_empty(&virt_timer_list))
 54		return 0;
 55	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
 56	return elapsed >= atomic64_read(&virt_timer_current);
 57}
 58
 59static void update_mt_scaling(void)
 60{
 61	u64 cycles_new[8], *cycles_old;
 62	u64 delta, fac, mult, div;
 63	int i;
 64
 65	stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
 66	cycles_old = this_cpu_ptr(mt_cycles);
 67	fac = 1;
 68	mult = div = 0;
 69	for (i = 0; i <= smp_cpu_mtid; i++) {
 70		delta = cycles_new[i] - cycles_old[i];
 71		div += delta;
 72		mult *= i + 1;
 73		mult += delta * fac;
 74		fac *= i + 1;
 75	}
 76	div *= fac;
 77	if (div > 0) {
 78		/* Update scaling factor */
 79		__this_cpu_write(mt_scaling_mult, mult);
 80		__this_cpu_write(mt_scaling_div, div);
 81		memcpy(cycles_old, cycles_new,
 82		       sizeof(u64) * (smp_cpu_mtid + 1));
 83	}
 84	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
 85}
 86
 87static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
 88{
 89	u64 delta;
 90
 91	delta = new - *tsk_vtime;
 92	*tsk_vtime = new;
 93	return delta;
 94}
 95
 96
 97static inline u64 scale_vtime(u64 vtime)
 98{
 99	u64 mult = __this_cpu_read(mt_scaling_mult);
100	u64 div = __this_cpu_read(mt_scaling_div);
101
102	if (smp_cpu_mtid)
103		return vtime * mult / div;
104	return vtime;
105}
106
107static void account_system_index_scaled(struct task_struct *p, u64 cputime,
108					enum cpu_usage_stat index)
109{
110	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
111	account_system_index_time(p, cputime_to_nsecs(cputime), index);
112}
113
114/*
115 * Update process times based on virtual cpu times stored by entry.S
116 * to the lowcore fields user_timer, system_timer & steal_clock.
117 */
118static int do_account_vtime(struct task_struct *tsk)
119{
120	u64 timer, clock, user, guest, system, hardirq, softirq;
121	struct lowcore *lc = get_lowcore();
122
123	timer = lc->last_update_timer;
124	clock = lc->last_update_clock;
125	asm volatile(
126		"	stpt	%0\n"	/* Store current cpu timer value */
127		"	stckf	%1"	/* Store current tod clock value */
128		: "=Q" (lc->last_update_timer),
129		  "=Q" (lc->last_update_clock)
130		: : "cc");
131	clock = lc->last_update_clock - clock;
132	timer -= lc->last_update_timer;
133
134	if (hardirq_count())
135		lc->hardirq_timer += timer;
136	else
137		lc->system_timer += timer;
138
139	/* Update MT utilization calculation */
140	if (smp_cpu_mtid &&
141	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
142		update_mt_scaling();
143
144	/* Calculate cputime delta */
145	user = update_tsk_timer(&tsk->thread.user_timer,
146				READ_ONCE(lc->user_timer));
147	guest = update_tsk_timer(&tsk->thread.guest_timer,
148				 READ_ONCE(lc->guest_timer));
149	system = update_tsk_timer(&tsk->thread.system_timer,
150				  READ_ONCE(lc->system_timer));
151	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
152				   READ_ONCE(lc->hardirq_timer));
153	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
154				   READ_ONCE(lc->softirq_timer));
155	lc->steal_timer +=
156		clock - user - guest - system - hardirq - softirq;
157
158	/* Push account value */
159	if (user) {
160		account_user_time(tsk, cputime_to_nsecs(user));
161		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
162	}
163
164	if (guest) {
165		account_guest_time(tsk, cputime_to_nsecs(guest));
166		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
167	}
168
169	if (system)
170		account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
171	if (hardirq)
172		account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
173	if (softirq)
174		account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
175
176	return virt_timer_forward(user + guest + system + hardirq + softirq);
177}
178
179void vtime_task_switch(struct task_struct *prev)
180{
181	struct lowcore *lc = get_lowcore();
182
183	do_account_vtime(prev);
184	prev->thread.user_timer = lc->user_timer;
185	prev->thread.guest_timer = lc->guest_timer;
186	prev->thread.system_timer = lc->system_timer;
187	prev->thread.hardirq_timer = lc->hardirq_timer;
188	prev->thread.softirq_timer = lc->softirq_timer;
189	lc->user_timer = current->thread.user_timer;
190	lc->guest_timer = current->thread.guest_timer;
191	lc->system_timer = current->thread.system_timer;
192	lc->hardirq_timer = current->thread.hardirq_timer;
193	lc->softirq_timer = current->thread.softirq_timer;
194}
195
196/*
197 * In s390, accounting pending user time also implies
198 * accounting system time in order to correctly compute
199 * the stolen time accounting.
200 */
201void vtime_flush(struct task_struct *tsk)
202{
203	struct lowcore *lc = get_lowcore();
204	u64 steal, avg_steal;
205
206	if (do_account_vtime(tsk))
207		virt_timer_expire();
208
209	steal = lc->steal_timer;
210	avg_steal = lc->avg_steal_timer;
211	if ((s64) steal > 0) {
212		lc->steal_timer = 0;
213		account_steal_time(cputime_to_nsecs(steal));
214		avg_steal += steal;
215	}
216	lc->avg_steal_timer = avg_steal / 2;
217}
218
219static u64 vtime_delta(void)
220{
221	struct lowcore *lc = get_lowcore();
222	u64 timer = lc->last_update_timer;
223
224	lc->last_update_timer = get_cpu_timer();
225	return timer - lc->last_update_timer;
226}
227
228void vtime_account_kernel(struct task_struct *tsk)
229{
230	struct lowcore *lc = get_lowcore();
231	u64 delta = vtime_delta();
232
233	if (tsk->flags & PF_VCPU)
234		lc->guest_timer += delta;
235	else
236		lc->system_timer += delta;
237}
238EXPORT_SYMBOL_GPL(vtime_account_kernel);
239
240void vtime_account_softirq(struct task_struct *tsk)
241{
242	get_lowcore()->softirq_timer += vtime_delta();
243}
244
245void vtime_account_hardirq(struct task_struct *tsk)
246{
247	get_lowcore()->hardirq_timer += vtime_delta();
248}
249
250/*
251 * Sorted add to a list. List is linear searched until first bigger
252 * element is found.
253 */
254static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
255{
256	struct vtimer_list *tmp;
257
258	list_for_each_entry(tmp, head, entry) {
259		if (tmp->expires > timer->expires) {
260			list_add_tail(&timer->entry, &tmp->entry);
261			return;
262		}
263	}
264	list_add_tail(&timer->entry, head);
265}
266
267/*
268 * Handler for expired virtual CPU timer.
269 */
270static void virt_timer_expire(void)
271{
272	struct vtimer_list *timer, *tmp;
273	unsigned long elapsed;
274	LIST_HEAD(cb_list);
275
276	/* walk timer list, fire all expired timers */
277	spin_lock(&virt_timer_lock);
278	elapsed = atomic64_read(&virt_timer_elapsed);
279	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
280		if (timer->expires < elapsed)
281			/* move expired timer to the callback queue */
282			list_move_tail(&timer->entry, &cb_list);
283		else
284			timer->expires -= elapsed;
285	}
286	if (!list_empty(&virt_timer_list)) {
287		timer = list_first_entry(&virt_timer_list,
288					 struct vtimer_list, entry);
289		atomic64_set(&virt_timer_current, timer->expires);
290	}
291	atomic64_sub(elapsed, &virt_timer_elapsed);
292	spin_unlock(&virt_timer_lock);
293
294	/* Do callbacks and recharge periodic timers */
295	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
296		list_del_init(&timer->entry);
297		timer->function(timer->data);
298		if (timer->interval) {
299			/* Recharge interval timer */
300			timer->expires = timer->interval +
301				atomic64_read(&virt_timer_elapsed);
302			spin_lock(&virt_timer_lock);
303			list_add_sorted(timer, &virt_timer_list);
304			spin_unlock(&virt_timer_lock);
305		}
306	}
307}
308
309void init_virt_timer(struct vtimer_list *timer)
310{
311	timer->function = NULL;
312	INIT_LIST_HEAD(&timer->entry);
313}
314EXPORT_SYMBOL(init_virt_timer);
315
316static inline int vtimer_pending(struct vtimer_list *timer)
317{
318	return !list_empty(&timer->entry);
319}
320
321static void internal_add_vtimer(struct vtimer_list *timer)
322{
323	if (list_empty(&virt_timer_list)) {
324		/* First timer, just program it. */
325		atomic64_set(&virt_timer_current, timer->expires);
326		atomic64_set(&virt_timer_elapsed, 0);
327		list_add(&timer->entry, &virt_timer_list);
328	} else {
329		/* Update timer against current base. */
330		timer->expires += atomic64_read(&virt_timer_elapsed);
331		if (likely((s64) timer->expires <
332			   (s64) atomic64_read(&virt_timer_current)))
333			/* The new timer expires before the current timer. */
334			atomic64_set(&virt_timer_current, timer->expires);
335		/* Insert new timer into the list. */
336		list_add_sorted(timer, &virt_timer_list);
337	}
338}
339
340static void __add_vtimer(struct vtimer_list *timer, int periodic)
341{
342	unsigned long flags;
343
344	timer->interval = periodic ? timer->expires : 0;
345	spin_lock_irqsave(&virt_timer_lock, flags);
346	internal_add_vtimer(timer);
347	spin_unlock_irqrestore(&virt_timer_lock, flags);
348}
349
350/*
351 * add_virt_timer - add a oneshot virtual CPU timer
352 */
353void add_virt_timer(struct vtimer_list *timer)
354{
355	__add_vtimer(timer, 0);
356}
357EXPORT_SYMBOL(add_virt_timer);
358
359/*
360 * add_virt_timer_int - add an interval virtual CPU timer
361 */
362void add_virt_timer_periodic(struct vtimer_list *timer)
363{
364	__add_vtimer(timer, 1);
365}
366EXPORT_SYMBOL(add_virt_timer_periodic);
367
368static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
369{
370	unsigned long flags;
371	int rc;
372
373	BUG_ON(!timer->function);
374
375	if (timer->expires == expires && vtimer_pending(timer))
376		return 1;
377	spin_lock_irqsave(&virt_timer_lock, flags);
378	rc = vtimer_pending(timer);
379	if (rc)
380		list_del_init(&timer->entry);
381	timer->interval = periodic ? expires : 0;
382	timer->expires = expires;
383	internal_add_vtimer(timer);
384	spin_unlock_irqrestore(&virt_timer_lock, flags);
385	return rc;
386}
387
388/*
389 * returns whether it has modified a pending timer (1) or not (0)
390 */
391int mod_virt_timer(struct vtimer_list *timer, u64 expires)
392{
393	return __mod_vtimer(timer, expires, 0);
394}
395EXPORT_SYMBOL(mod_virt_timer);
396
397/*
398 * returns whether it has modified a pending timer (1) or not (0)
399 */
400int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
401{
402	return __mod_vtimer(timer, expires, 1);
403}
404EXPORT_SYMBOL(mod_virt_timer_periodic);
405
406/*
407 * Delete a virtual timer.
408 *
409 * returns whether the deleted timer was pending (1) or not (0)
410 */
411int del_virt_timer(struct vtimer_list *timer)
412{
413	unsigned long flags;
414
415	if (!vtimer_pending(timer))
416		return 0;
417	spin_lock_irqsave(&virt_timer_lock, flags);
418	list_del_init(&timer->entry);
419	spin_unlock_irqrestore(&virt_timer_lock, flags);
420	return 1;
421}
422EXPORT_SYMBOL(del_virt_timer);
423
424/*
425 * Start the virtual CPU timer on the current CPU.
426 */
427void vtime_init(void)
428{
429	/* set initial cpu timer */
430	set_vtimer(VTIMER_MAX_SLICE);
431	/* Setup initial MT scaling values */
432	if (smp_cpu_mtid) {
433		__this_cpu_write(mt_scaling_jiffies, jiffies);
434		__this_cpu_write(mt_scaling_mult, 1);
435		__this_cpu_write(mt_scaling_div, 1);
436		stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
437	}
438}