cputime.c 22.4 KB
Newer Older
1 2 3 4 5
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/tsacct_kern.h>
#include <linux/kernel_stat.h>
#include <linux/static_key.h>
6
#include <linux/context_tracking.h>
7
#include <linux/sched/cputime.h>
8 9 10 11 12 13
#include "sched.h"

#ifdef CONFIG_IRQ_TIME_ACCOUNTING

/*
 * There are no locks covering percpu hardirq/softirq time.
14
 * They are only modified in vtime_account, on corresponding CPU
15 16 17
 * with interrupts disabled. So, writes are safe.
 * They are read and saved off onto struct rq in update_rq_clock().
 * This may result in other CPU reading this CPU's irq time and can
18
 * race with irq/vtime_account on this CPU. We would either get old
19 20 21 22
 * or new value with a side effect of accounting a slice of irq time to wrong
 * task when irq is in progress while we read rq->clock. That is a worthy
 * compromise in place of having locks on each irq in account_system_time.
 */
23
DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
24 25 26 27 28 29 30 31 32 33 34 35 36

static int sched_clock_irqtime;

void enable_sched_clock_irqtime(void)
{
	sched_clock_irqtime = 1;
}

void disable_sched_clock_irqtime(void)
{
	sched_clock_irqtime = 0;
}

37 38 39 40 41 42 43 44 45 46 47 48
static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
				  enum cpu_usage_stat idx)
{
	u64 *cpustat = kcpustat_this_cpu->cpustat;

	u64_stats_update_begin(&irqtime->sync);
	cpustat[idx] += delta;
	irqtime->total += delta;
	irqtime->tick_delta += delta;
	u64_stats_update_end(&irqtime->sync);
}

49 50 51 52
/*
 * Called before incrementing preempt_count on {soft,}irq_enter
 * and before decrementing preempt_count on {soft,}irq_exit.
 */
53
void irqtime_account_irq(struct task_struct *curr)
54
{
55
	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
56 57 58 59 60 61 62
	s64 delta;
	int cpu;

	if (!sched_clock_irqtime)
		return;

	cpu = smp_processor_id();
63 64
	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
	irqtime->irq_start_time += delta;
65 66 67 68 69 70 71

	/*
	 * We do not account for softirq time from ksoftirqd here.
	 * We want to continue accounting softirq time to ksoftirqd thread
	 * in that case, so as not to confuse scheduler with a special task
	 * that do not consume any time, but still wants to run.
	 */
72 73 74 75
	if (hardirq_count())
		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
	else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
76
}
77
EXPORT_SYMBOL_GPL(irqtime_account_irq);
78

79
static u64 irqtime_tick_accounted(u64 maxtime)
80
{
81
	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
82
	u64 delta;
83

84 85
	delta = min(irqtime->tick_delta, maxtime);
	irqtime->tick_delta -= delta;
86

87
	return delta;
88 89 90 91 92 93
}

#else /* CONFIG_IRQ_TIME_ACCOUNTING */

#define sched_clock_irqtime	(0)

94
static u64 irqtime_tick_accounted(u64 dummy)
95 96 97 98
{
	return 0;
}

99 100 101 102 103 104 105 106 107 108 109
#endif /* !CONFIG_IRQ_TIME_ACCOUNTING */

static inline void task_group_account_field(struct task_struct *p, int index,
					    u64 tmp)
{
	/*
	 * Since all updates are sure to touch the root cgroup, we
	 * get ourselves ahead and touch it first. If the root cgroup
	 * is the only cgroup, then nothing else should be necessary.
	 *
	 */
110
	__this_cpu_add(kernel_cpustat.cpustat[index], tmp);
111

112
	cpuacct_account_field(p, index, tmp);
113 114 115 116 117 118 119
}

/*
 * Account user cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in user space since the last update
 */
120
void account_user_time(struct task_struct *p, u64 cputime)
121 122 123 124
{
	int index;

	/* Add user time to process. */
125 126
	p->utime += cputime;
	account_group_user_time(p, cputime);
127

128
	index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
129 130

	/* Add user time to cpustat. */
131
	task_group_account_field(p, index, cputime);
132 133

	/* Account for user time used */
134
	acct_account_cputime(p);
135 136 137 138 139 140 141
}

/*
 * Account guest cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in virtual machine since the last update
 */
142
void account_guest_time(struct task_struct *p, u64 cputime)
143 144 145 146
{
	u64 *cpustat = kcpustat_this_cpu->cpustat;

	/* Add guest time to process. */
147 148 149
	p->utime += cputime;
	account_group_user_time(p, cputime);
	p->gtime += cputime;
150 151

	/* Add guest time to cpustat. */
152
	if (task_nice(p) > 0) {
153 154
		cpustat[CPUTIME_NICE] += cputime;
		cpustat[CPUTIME_GUEST_NICE] += cputime;
155
	} else {
156 157
		cpustat[CPUTIME_USER] += cputime;
		cpustat[CPUTIME_GUEST] += cputime;
158 159 160 161 162 163 164
	}
}

/*
 * Account system cpu time to a process and desired cpustat field
 * @p: the process that the cpu time gets accounted to
 * @cputime: the cpu time spent in kernel space since the last update
165
 * @index: pointer to cpustat field that has to be updated
166
 */
167
void account_system_index_time(struct task_struct *p,
168
			       u64 cputime, enum cpu_usage_stat index)
169 170
{
	/* Add system time to process. */
171 172
	p->stime += cputime;
	account_group_system_time(p, cputime);
173 174

	/* Add system time to cpustat. */
175
	task_group_account_field(p, index, cputime);
176 177

	/* Account for system time used */
178
	acct_account_cputime(p);
179 180 181 182 183 184 185 186
}

/*
 * Account system cpu time to a process.
 * @p: the process that the cpu time gets accounted to
 * @hardirq_offset: the offset to subtract from hardirq_count()
 * @cputime: the cpu time spent in kernel space since the last update
 */
187
void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
188 189 190 191
{
	int index;

	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
192
		account_guest_time(p, cputime);
193 194 195 196 197 198 199 200 201 202
		return;
	}

	if (hardirq_count() - hardirq_offset)
		index = CPUTIME_IRQ;
	else if (in_serving_softirq())
		index = CPUTIME_SOFTIRQ;
	else
		index = CPUTIME_SYSTEM;

203
	account_system_index_time(p, cputime, index);
204 205 206 207 208 209
}

/*
 * Account for involuntary wait time.
 * @cputime: the cpu time spent in involuntary wait
 */
210
void account_steal_time(u64 cputime)
211 212 213
{
	u64 *cpustat = kcpustat_this_cpu->cpustat;

214
	cpustat[CPUTIME_STEAL] += cputime;
215 216 217 218 219 220
}

/*
 * Account for idle time.
 * @cputime: the cpu time spent in idle wait
 */
221
void account_idle_time(u64 cputime)
222 223 224 225 226
{
	u64 *cpustat = kcpustat_this_cpu->cpustat;
	struct rq *rq = this_rq();

	if (atomic_read(&rq->nr_iowait) > 0)
227
		cpustat[CPUTIME_IOWAIT] += cputime;
228
	else
229
		cpustat[CPUTIME_IDLE] += cputime;
230 231
}

232 233 234 235 236
/*
 * When a guest is interrupted for a longer amount of time, missed clock
 * ticks are not redelivered later. Due to that, this function may on
 * occasion account more time than the calling functions think elapsed.
 */
237
static __always_inline u64 steal_account_process_time(u64 maxtime)
238 239 240
{
#ifdef CONFIG_PARAVIRT
	if (static_key_false(&paravirt_steal_enabled)) {
241
		u64 steal;
242 243 244

		steal = paravirt_steal_clock(smp_processor_id());
		steal -= this_rq()->prev_steal_time;
245 246 247
		steal = min(steal, maxtime);
		account_steal_time(steal);
		this_rq()->prev_steal_time += steal;
248

249
		return steal;
250 251
	}
#endif
252
	return 0;
253 254
}

255 256 257
/*
 * Account how much elapsed time was spent in steal, irq, or softirq time.
 */
258
static inline u64 account_other_time(u64 max)
259
{
260
	u64 accounted;
261

262 263 264
	/* Shall be converted to a lockdep-enabled lightweight check */
	WARN_ON_ONCE(!irqs_disabled());

265 266 267
	accounted = steal_account_process_time(max);

	if (accounted < max)
268
		accounted += irqtime_tick_accounted(max - accounted);
269 270 271 272

	return accounted;
}

273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292
#ifdef CONFIG_64BIT
static inline u64 read_sum_exec_runtime(struct task_struct *t)
{
	return t->se.sum_exec_runtime;
}
#else
static u64 read_sum_exec_runtime(struct task_struct *t)
{
	u64 ns;
	struct rq_flags rf;
	struct rq *rq;

	rq = task_rq_lock(t, &rf);
	ns = t->se.sum_exec_runtime;
	task_rq_unlock(rq, t, &rf);

	return ns;
}
#endif

293 294 295 296 297 298 299
/*
 * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
 * tasks (sum on group iteration) belonging to @tsk's group.
 */
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
{
	struct signal_struct *sig = tsk->signal;
300
	u64 utime, stime;
301
	struct task_struct *t;
302
	unsigned int seq, nextseq;
303
	unsigned long flags;
304

305 306 307 308 309 310 311 312 313 314 315
	/*
	 * Update current task runtime to account pending time since last
	 * scheduler action or thread_group_cputime() call. This thread group
	 * might have other running tasks on different CPUs, but updating
	 * their runtime can affect syscall performance, so we skip account
	 * those pending times and rely only on values updated on tick or
	 * other scheduler action.
	 */
	if (same_thread_group(current, tsk))
		(void) task_sched_runtime(current);

316
	rcu_read_lock();
317 318 319 320
	/* Attempt a lockless read on the first round. */
	nextseq = 0;
	do {
		seq = nextseq;
321
		flags = read_seqbegin_or_lock_irqsave(&sig->stats_lock, &seq);
322 323 324 325 326 327 328 329
		times->utime = sig->utime;
		times->stime = sig->stime;
		times->sum_exec_runtime = sig->sum_sched_runtime;

		for_each_thread(tsk, t) {
			task_cputime(t, &utime, &stime);
			times->utime += utime;
			times->stime += stime;
330
			times->sum_exec_runtime += read_sum_exec_runtime(t);
331 332 333 334
		}
		/* If lockless access failed, take the lock. */
		nextseq = 1;
	} while (need_seqretry(&sig->stats_lock, seq));
335
	done_seqretry_irqrestore(&sig->stats_lock, seq, flags);
336 337 338
	rcu_read_unlock();
}

339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
/*
 * Account a tick to a process and cpustat
 * @p: the process that the cpu time gets accounted to
 * @user_tick: is the tick from userspace
 * @rq: the pointer to rq
 *
 * Tick demultiplexing follows the order
 * - pending hardirq update
 * - pending softirq update
 * - user_time
 * - idle_time
 * - system time
 *   - check for guest_time
 *   - else account as system_time
 *
 * Check for hardirq is done both for system and user time as there is
 * no timer going off while we are on hardirq and hence we may never get an
 * opportunity to update it solely in system time.
 * p->stime and friends are only updated on system time and not on irq
 * softirq as those do not count in task exec_runtime any more.
 */
static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
362
					 struct rq *rq, int ticks)
363
{
364
	u64 other, cputime = TICK_NSEC * ticks;
365

366 367 368 369 370 371 372
	/*
	 * When returning from idle, many ticks can get accounted at
	 * once, including some ticks of steal, irq, and softirq time.
	 * Subtract those ticks from the amount of time accounted to
	 * idle, or potentially user or system time. Due to rounding,
	 * other time can exceed ticks occasionally.
	 */
373
	other = account_other_time(ULONG_MAX);
374
	if (other >= cputime)
375
		return;
376

377
	cputime -= other;
378

379
	if (this_cpu_ksoftirqd() == p) {
380 381 382 383 384
		/*
		 * ksoftirqd time do not get accounted in cpu_softirq_time.
		 * So, we have to handle it separately here.
		 * Also, p->stime needs to be updated for ksoftirqd.
		 */
385
		account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
386
	} else if (user_tick) {
387
		account_user_time(p, cputime);
388
	} else if (p == rq->idle) {
389
		account_idle_time(cputime);
390
	} else if (p->flags & PF_VCPU) { /* System time or guest time */
391
		account_guest_time(p, cputime);
392
	} else {
393
		account_system_index_time(p, cputime, CPUTIME_SYSTEM);
394 395 396 397 398 399 400
	}
}

static void irqtime_account_idle_ticks(int ticks)
{
	struct rq *rq = this_rq();

401
	irqtime_account_process_tick(current, 0, rq, ticks);
402 403
}
#else /* CONFIG_IRQ_TIME_ACCOUNTING */
404 405
static inline void irqtime_account_idle_ticks(int ticks) {}
static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
406
						struct rq *rq, int nr_ticks) {}
407 408 409 410 411 412
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */

/*
 * Use precise platform statistics if available:
 */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
413

414
#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
415
void vtime_common_task_switch(struct task_struct *prev)
416 417 418 419 420 421
{
	if (is_idle_task(prev))
		vtime_account_idle(prev);
	else
		vtime_account_system(prev);

422
	vtime_flush(prev);
423 424 425
	arch_vtime_task_switch(prev);
}
#endif
426

427 428 429 430
#endif /* CONFIG_VIRT_CPU_ACCOUNTING */


#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
431 432 433
/*
 * Archs that account the whole time spent in the idle task
 * (outside irq) as idle time can rely on this and just implement
434
 * vtime_account_system() and vtime_account_idle(). Archs that
435 436 437 438 439
 * have other meaning of the idle time (s390 only includes the
 * time spent by the CPU when it's in low power mode) must override
 * vtime_account().
 */
#ifndef __ARCH_HAS_VTIME_ACCOUNT
440
void vtime_account_irq_enter(struct task_struct *tsk)
441
{
442 443 444 445
	if (!in_interrupt() && is_idle_task(tsk))
		vtime_account_idle(tsk);
	else
		vtime_account_system(tsk);
446
}
447
EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
448
#endif /* __ARCH_HAS_VTIME_ACCOUNT */
449

450
void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
451 452 453 454
{
	*ut = p->utime;
	*st = p->stime;
}
455
EXPORT_SYMBOL_GPL(task_cputime_adjusted);
456

457
void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
458 459
{
	struct task_cputime cputime;
460

461 462 463 464 465 466 467 468 469 470 471 472
	thread_group_cputime(p, &cputime);

	*ut = cputime.utime;
	*st = cputime.stime;
}
#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
/*
 * Account a single tick of cpu time.
 * @p: the process that the cpu time gets accounted to
 * @user_tick: indicates if the tick is a user or a system tick
 */
void account_process_tick(struct task_struct *p, int user_tick)
473
{
474
	u64 cputime, steal;
475
	struct rq *rq = this_rq();
476

477
	if (vtime_accounting_cpu_enabled())
478 479 480
		return;

	if (sched_clock_irqtime) {
481
		irqtime_account_process_tick(p, user_tick, rq, 1);
482 483 484
		return;
	}

485
	cputime = TICK_NSEC;
486
	steal = steal_account_process_time(ULONG_MAX);
487

488
	if (steal >= cputime)
489
		return;
490

491
	cputime -= steal;
492

493
	if (user_tick)
494
		account_user_time(p, cputime);
495
	else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET))
496
		account_system_time(p, HARDIRQ_OFFSET, cputime);
497
	else
498
		account_idle_time(cputime);
499
}
500

501 502 503 504 505 506
/*
 * Account multiple ticks of idle time.
 * @ticks: number of stolen ticks
 */
void account_idle_ticks(unsigned long ticks)
{
507
	u64 cputime, steal;
508

509 510 511 512 513
	if (sched_clock_irqtime) {
		irqtime_account_idle_ticks(ticks);
		return;
	}

514
	cputime = ticks * TICK_NSEC;
515
	steal = steal_account_process_time(ULONG_MAX);
516 517 518 519 520 521

	if (steal >= cputime)
		return;

	cputime -= steal;
	account_idle_time(cputime);
522
}
523

524
/*
525 526
 * Perform (stime * rtime) / total, but avoid multiplication overflow by
 * loosing precision when the numbers are big.
527
 */
528
static u64 scale_stime(u64 stime, u64 rtime, u64 total)
529
{
530
	u64 scaled;
531

532 533
	for (;;) {
		/* Make sure "rtime" is the bigger of stime/rtime */
534 535
		if (stime > rtime)
			swap(rtime, stime);
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557

		/* Make sure 'total' fits in 32 bits */
		if (total >> 32)
			goto drop_precision;

		/* Does rtime (and thus stime) fit in 32 bits? */
		if (!(rtime >> 32))
			break;

		/* Can we just balance rtime/stime rather than dropping bits? */
		if (stime >> 31)
			goto drop_precision;

		/* We can grow stime and shrink rtime and try to make them both fit */
		stime <<= 1;
		rtime >>= 1;
		continue;

drop_precision:
		/* We drop from rtime, it has more bits than stime */
		rtime >>= 1;
		total >>= 1;
558
	}
559

560 561 562 563 564
	/*
	 * Make sure gcc understands that this is a 32x32->64 multiply,
	 * followed by a 64/32->64 divide.
	 */
	scaled = div_u64((u64) (u32) stime * (u64) (u32) rtime, (u32)total);
565
	return scaled;
566 567
}

568
/*
569 570
 * Adjust tick based cputime random precision against scheduler runtime
 * accounting.
571
 *
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
 * Tick based cputime accounting depend on random scheduling timeslices of a
 * task to be interrupted or not by the timer.  Depending on these
 * circumstances, the number of these interrupts may be over or
 * under-optimistic, matching the real user and system cputime with a variable
 * precision.
 *
 * Fix this by scaling these tick based values against the total runtime
 * accounted by the CFS scheduler.
 *
 * This code provides the following guarantees:
 *
 *   stime + utime == rtime
 *   stime_i+1 >= stime_i, utime_i+1 >= utime_i
 *
 * Assuming that rtime_i+1 >= rtime_i.
587
 */
588
static void cputime_adjust(struct task_cputime *curr,
589
			   struct prev_cputime *prev,
590
			   u64 *ut, u64 *st)
591
{
592
	u64 rtime, stime, utime;
593
	unsigned long flags;
594

595 596
	/* Serialize concurrent callers such that we can honour our guarantees */
	raw_spin_lock_irqsave(&prev->lock, flags);
597
	rtime = curr->sum_exec_runtime;
598

599
	/*
600 601 602 603 604 605
	 * This is possible under two circumstances:
	 *  - rtime isn't monotonic after all (a bug);
	 *  - we got reordered by the lock.
	 *
	 * In both cases this acts as a filter such that the rest of the code
	 * can assume it is monotonic regardless of anything else.
606 607 608 609
	 */
	if (prev->stime + prev->utime >= rtime)
		goto out;

610 611 612
	stime = curr->stime;
	utime = curr->utime;

613
	/*
614 615 616
	 * If either stime or utime are 0, assume all runtime is userspace.
	 * Once a task gets some ticks, the monotonicy code at 'update:'
	 * will ensure things converge to the observed ratio.
617
	 */
618 619 620
	if (stime == 0) {
		utime = rtime;
		goto update;
621
	}
622

623 624 625 626 627 628 629 630
	if (utime == 0) {
		stime = rtime;
		goto update;
	}

	stime = scale_stime(stime, rtime, stime + utime);

update:
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
	/*
	 * Make sure stime doesn't go backwards; this preserves monotonicity
	 * for utime because rtime is monotonic.
	 *
	 *  utime_i+1 = rtime_i+1 - stime_i
	 *            = rtime_i+1 - (rtime_i - utime_i)
	 *            = (rtime_i+1 - rtime_i) + utime_i
	 *            >= utime_i
	 */
	if (stime < prev->stime)
		stime = prev->stime;
	utime = rtime - stime;

	/*
	 * Make sure utime doesn't go backwards; this still preserves
	 * monotonicity for stime, analogous argument to above.
	 */
	if (utime < prev->utime) {
		utime = prev->utime;
		stime = rtime - utime;
	}
652

653 654
	prev->stime = stime;
	prev->utime = utime;
655
out:
656 657
	*ut = prev->utime;
	*st = prev->stime;
658
	raw_spin_unlock_irqrestore(&prev->lock, flags);
659
}
660

661
void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
662 663 664 665 666
{
	struct task_cputime cputime = {
		.sum_exec_runtime = p->se.sum_exec_runtime,
	};

667
	task_cputime(p, &cputime.utime, &cputime.stime);
668
	cputime_adjust(&cputime, &p->prev_cputime, ut, st);
669
}
670
EXPORT_SYMBOL_GPL(task_cputime_adjusted);
671

672
void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
673 674 675 676
{
	struct task_cputime cputime;

	thread_group_cputime(p, &cputime);
677
	cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
678
}
679
#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
680 681

#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
682
static u64 vtime_delta(struct vtime *vtime)
683
{
684
	unsigned long long clock;
685

686
	clock = sched_clock();
687
	if (clock < vtime->starttime)
688
		return 0;
689

690
	return clock - vtime->starttime;
691 692
}

693
static u64 get_vtime_delta(struct vtime *vtime)
694
{
695 696
	u64 delta = vtime_delta(vtime);
	u64 other;
697

698 699 700 701 702 703 704
	/*
	 * Unlike tick based timing, vtime based timing never has lost
	 * ticks, and no need for steal time accounting to make up for
	 * lost ticks. Vtime accounts a rounded version of actual
	 * elapsed time. Limit account_other_time to prevent rounding
	 * errors from causing elapsed vtime to go negative.
	 */
705
	other = account_other_time(delta);
706
	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
707
	vtime->starttime += delta;
708

709
	return delta - other;
710 711
}

712 713
static void __vtime_account_system(struct task_struct *tsk,
				   struct vtime *vtime)
714
{
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
	vtime->stime += get_vtime_delta(vtime);
	if (vtime->stime >= TICK_NSEC) {
		account_system_time(tsk, irq_count(), vtime->stime);
		vtime->stime = 0;
	}
}

static void vtime_account_guest(struct task_struct *tsk,
				struct vtime *vtime)
{
	vtime->gtime += get_vtime_delta(vtime);
	if (vtime->gtime >= TICK_NSEC) {
		account_guest_time(tsk, vtime->gtime);
		vtime->gtime = 0;
	}
730 731
}

732 733
void vtime_account_system(struct task_struct *tsk)
{
734 735 736
	struct vtime *vtime = &tsk->vtime;

	if (!vtime_delta(vtime))
737 738
		return;

739
	write_seqcount_begin(&vtime->seqcount);
740 741 742 743 744
	/* We might have scheduled out from guest path */
	if (current->flags & PF_VCPU)
		vtime_account_guest(tsk, vtime);
	else
		__vtime_account_system(tsk, vtime);
745
	write_seqcount_end(&vtime->seqcount);
746
}
747

748
void vtime_user_enter(struct task_struct *tsk)
749
{
750 751 752
	struct vtime *vtime = &tsk->vtime;

	write_seqcount_begin(&vtime->seqcount);
753
	__vtime_account_system(tsk, vtime);
754 755
	vtime->state = VTIME_USER;
	write_seqcount_end(&vtime->seqcount);
756 757
}

758
void vtime_user_exit(struct task_struct *tsk)
759
{
760 761 762
	struct vtime *vtime = &tsk->vtime;

	write_seqcount_begin(&vtime->seqcount);
763 764 765 766 767
	vtime->utime += get_vtime_delta(vtime);
	if (vtime->utime >= TICK_NSEC) {
		account_user_time(tsk, vtime->utime);
		vtime->utime = 0;
	}
768 769
	vtime->state = VTIME_SYS;
	write_seqcount_end(&vtime->seqcount);
770 771 772 773
}

void vtime_guest_enter(struct task_struct *tsk)
{
774
	struct vtime *vtime = &tsk->vtime;
775 776
	/*
	 * The flags must be updated under the lock with
777
	 * the vtime_starttime flush and update.
778 779 780 781
	 * That enforces a right ordering and update sequence
	 * synchronization against the reader (task_gtime())
	 * that can thus safely catch up with a tickless delta.
	 */
782
	write_seqcount_begin(&vtime->seqcount);
783
	__vtime_account_system(tsk, vtime);
784
	current->flags |= PF_VCPU;
785
	write_seqcount_end(&vtime->seqcount);
786
}
787
EXPORT_SYMBOL_GPL(vtime_guest_enter);
788 789 790

void vtime_guest_exit(struct task_struct *tsk)
{
791 792 793
	struct vtime *vtime = &tsk->vtime;

	write_seqcount_begin(&vtime->seqcount);
794
	vtime_account_guest(tsk, vtime);
795
	current->flags &= ~PF_VCPU;
796
	write_seqcount_end(&vtime->seqcount);
797
}
798
EXPORT_SYMBOL_GPL(vtime_guest_exit);
799 800 801

void vtime_account_idle(struct task_struct *tsk)
{
802
	account_idle_time(get_vtime_delta(&tsk->vtime));
803
}
804

805 806
void arch_vtime_task_switch(struct task_struct *prev)
{
807
	struct vtime *vtime = &prev->vtime;
808

809 810 811 812 813 814 815 816
	write_seqcount_begin(&vtime->seqcount);
	vtime->state = VTIME_INACTIVE;
	write_seqcount_end(&vtime->seqcount);

	vtime = &current->vtime;

	write_seqcount_begin(&vtime->seqcount);
	vtime->state = VTIME_SYS;
817
	vtime->starttime = sched_clock();
818
	write_seqcount_end(&vtime->seqcount);
819 820
}

821
void vtime_init_idle(struct task_struct *t, int cpu)
822
{
823
	struct vtime *vtime = &t->vtime;
824 825
	unsigned long flags;

826
	local_irq_save(flags);
827 828
	write_seqcount_begin(&vtime->seqcount);
	vtime->state = VTIME_SYS;
829
	vtime->starttime = sched_clock();
830
	write_seqcount_end(&vtime->seqcount);
831
	local_irq_restore(flags);
832 833
}

834
u64 task_gtime(struct task_struct *t)
835
{
836
	struct vtime *vtime = &t->vtime;
837
	unsigned int seq;
838
	u64 gtime;
839

840
	if (!vtime_accounting_enabled())
841 842
		return t->gtime;

843
	do {
844
		seq = read_seqcount_begin(&vtime->seqcount);
845 846

		gtime = t->gtime;
847
		if (vtime->state == VTIME_SYS && t->flags & PF_VCPU)
848
			gtime += vtime->gtime + vtime_delta(vtime);
849

850
	} while (read_seqcount_retry(&vtime->seqcount, seq));
851 852 853 854 855 856 857 858 859

	return gtime;
}

/*
 * Fetch cputime raw values from fields of task_struct and
 * add up the pending nohz execution time since the last
 * cputime snapshot.
 */
860
void task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
861
{
862
	struct vtime *vtime = &t->vtime;
863
	unsigned int seq;
864
	u64 delta;
865

866 867 868 869 870
	if (!vtime_accounting_enabled()) {
		*utime = t->utime;
		*stime = t->stime;
		return;
	}
871

872
	do {
873
		seq = read_seqcount_begin(&vtime->seqcount);
874

875 876
		*utime = t->utime;
		*stime = t->stime;
877 878

		/* Task is sleeping, nothing to add */
879
		if (vtime->state == VTIME_INACTIVE || is_idle_task(t))
880 881
			continue;

882
		delta = vtime_delta(vtime);
883 884 885 886 887

		/*
		 * Task runs either in user or kernel space, add pending nohz time to
		 * the right place.
		 */
888
		if (vtime->state == VTIME_USER || t->flags & PF_VCPU)
889
			*utime += vtime->utime + delta;
890
		else if (vtime->state == VTIME_SYS)
891
			*stime += vtime->stime + delta;
892
	} while (read_seqcount_retry(&vtime->seqcount, seq));
893
}
894
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */