Commit cf7a617d authored by Philippe Gerum's avatar Philippe Gerum

clocksource, clockevents: ipipe: enable pipelined ticks and timekeeping

parent c30f084f
...@@ -129,6 +129,15 @@ struct clock_event_device { ...@@ -129,6 +129,15 @@ struct clock_event_device {
const struct cpumask *cpumask; const struct cpumask *cpumask;
struct list_head list; struct list_head list;
struct module *owner; struct module *owner;
#ifdef CONFIG_IPIPE
struct ipipe_timer *ipipe_timer;
unsigned ipipe_stolen;
#define clockevent_ipipe_stolen(evt) ((evt)->ipipe_stolen)
#else
#define clockevent_ipipe_stolen(evt) (0)
#endif /* !CONFIG_IPIPE */
} ____cacheline_aligned; } ____cacheline_aligned;
/* Helpers to verify state of a clockevent device */ /* Helpers to verify state of a clockevent device */
......
...@@ -107,6 +107,9 @@ struct clocksource { ...@@ -107,6 +107,9 @@ struct clocksource {
u64 wd_last; u64 wd_last;
#endif #endif
struct module *owner; struct module *owner;
#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE
u64 (*ipipe_read)(struct clocksource *cs);
#endif /* CONFIG_IPIPE_WANT_CLOCKSOURCE */
}; };
/* /*
......
...@@ -135,7 +135,7 @@ extern void update_vsyscall_tz(void); ...@@ -135,7 +135,7 @@ extern void update_vsyscall_tz(void);
#elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD) #elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD)
extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm,
struct clocksource *c, u32 mult, struct clocksource *c, u32 mult, u32 shift,
u64 cycle_last); u64 cycle_last);
extern void update_vsyscall_tz(void); extern void update_vsyscall_tz(void);
......
...@@ -347,5 +347,13 @@ extern void read_boot_clock64(struct timespec64 *ts); ...@@ -347,5 +347,13 @@ extern void read_boot_clock64(struct timespec64 *ts);
extern int update_persistent_clock(struct timespec now); extern int update_persistent_clock(struct timespec now);
extern int update_persistent_clock64(struct timespec64 now); extern int update_persistent_clock64(struct timespec64 now);
#ifdef CONFIG_IPIPE
void update_root_process_times(struct pt_regs *regs);
#else /* !CONFIG_IPIPE */
static inline void update_root_process_times(struct pt_regs *regs)
{
update_process_times(user_mode(regs));
}
#endif /* CONFIG_IPIPE */
#endif #endif
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
#include <linux/module.h> #include <linux/module.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/device.h> #include <linux/device.h>
#include <linux/ipipe_tickdev.h>
#include "tick-internal.h" #include "tick-internal.h"
...@@ -453,6 +454,8 @@ void clockevents_register_device(struct clock_event_device *dev) ...@@ -453,6 +454,8 @@ void clockevents_register_device(struct clock_event_device *dev)
/* Initialize state to DETACHED */ /* Initialize state to DETACHED */
clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED); clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
ipipe_host_timer_register(dev);
if (!dev->cpumask) { if (!dev->cpumask) {
WARN_ON(num_possible_cpus() > 1); WARN_ON(num_possible_cpus() > 1);
dev->cpumask = cpumask_of(smp_processor_id()); dev->cpumask = cpumask_of(smp_processor_id());
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
#include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */ #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
#include <linux/tick.h> #include <linux/tick.h>
#include <linux/kthread.h> #include <linux/kthread.h>
#include <linux/kallsyms.h>
#include "tick-internal.h" #include "tick-internal.h"
#include "timekeeping_internal.h" #include "timekeeping_internal.h"
...@@ -177,6 +178,9 @@ static void clocksource_watchdog(unsigned long data) ...@@ -177,6 +178,9 @@ static void clocksource_watchdog(unsigned long data)
u64 csnow, wdnow, cslast, wdlast, delta; u64 csnow, wdnow, cslast, wdlast, delta;
int64_t wd_nsec, cs_nsec; int64_t wd_nsec, cs_nsec;
int next_cpu, reset_pending; int next_cpu, reset_pending;
#ifdef CONFIG_IPIPE
u64 wdref;
#endif
spin_lock(&watchdog_lock); spin_lock(&watchdog_lock);
if (!watchdog_running) if (!watchdog_running)
...@@ -193,11 +197,24 @@ static void clocksource_watchdog(unsigned long data) ...@@ -193,11 +197,24 @@ static void clocksource_watchdog(unsigned long data)
continue; continue;
} }
#ifdef CONFIG_IPIPE
retry:
#endif
local_irq_disable(); local_irq_disable();
#ifdef CONFIG_IPIPE
wdref = watchdog->read(watchdog);
#endif
csnow = cs->read(cs); csnow = cs->read(cs);
wdnow = watchdog->read(watchdog); wdnow = watchdog->read(watchdog);
local_irq_enable(); local_irq_enable();
#ifdef CONFIG_IPIPE
wd_nsec = clocksource_cyc2ns((wdnow - wdref) & watchdog->mask,
watchdog->mult, watchdog->shift);
if (wd_nsec > WATCHDOG_THRESHOLD)
goto retry;
#endif
/* Clocksource initialized ? */ /* Clocksource initialized ? */
if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
atomic_read(&watchdog_reset_pending)) { atomic_read(&watchdog_reset_pending)) {
...@@ -678,6 +695,95 @@ static int __init clocksource_done_booting(void) ...@@ -678,6 +695,95 @@ static int __init clocksource_done_booting(void)
} }
fs_initcall(clocksource_done_booting); fs_initcall(clocksource_done_booting);
#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE
unsigned long long __ipipe_cs_freq;
EXPORT_SYMBOL_GPL(__ipipe_cs_freq);
struct clocksource *__ipipe_cs;
EXPORT_SYMBOL_GPL(__ipipe_cs);
u64 (*__ipipe_cs_read)(struct clocksource *cs);
u64 __ipipe_cs_last_tsc;
u64 __ipipe_cs_mask;
unsigned __ipipe_cs_lat = 0xffffffff;
static void ipipe_check_clocksource(struct clocksource *cs)
{
u64 (*cread)(struct clocksource *cs);
u64 lat, mask, saved;
unsigned long long freq;
unsigned long flags;
unsigned i;
if (cs->ipipe_read) {
mask = CLOCKSOURCE_MASK(64);
cread = cs->ipipe_read;
} else {
mask = cs->mask;
cread = cs->read;
if ((cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) == 0)
return;
/*
* We only support masks such that cs->mask + 1 is a power of 2,
* 64 bits masks or masks lesser than 32 bits
*/
if (mask != CLOCKSOURCE_MASK(64)
&& ((mask & (mask + 1)) != 0 || mask > 0xffffffff))
return;
}
/*
* We prefer a clocksource with a better resolution than 1us
*/
if (cs->shift <= 34) {
freq = 1000000000ULL << cs->shift;
do_div(freq, cs->mult);
} else {
freq = 1000000ULL << cs->shift;
do_div(freq, cs->mult);
freq *= 1000;
}
if (freq < 1000000)
return;
/* Measure the clocksource latency */
flags = hard_local_irq_save();
saved = __ipipe_cs_last_tsc;
lat = cread(cs);
for (i = 0; i < 10; i++)
cread(cs);
lat = cread(cs) - lat;
__ipipe_cs_last_tsc = saved;
hard_local_irq_restore(flags);
lat = (lat * cs->mult) >> cs->shift;
do_div(lat, i + 1);
if (!strcmp(cs->name, override_name))
goto skip_tests;
if (lat > __ipipe_cs_lat)
return;
if (__ipipe_cs && !strcmp(__ipipe_cs->name, override_name))
return;
skip_tests:
flags = hard_local_irq_save();
if (__ipipe_cs_last_tsc == 0) {
__ipipe_cs_lat = lat;
__ipipe_cs_freq = freq;
__ipipe_cs = cs;
__ipipe_cs_read = cread;
__ipipe_cs_mask = mask;
}
hard_local_irq_restore(flags);
}
#else /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */
#define ipipe_check_clocksource(cs) do { }while (0)
#endif /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */
/* /*
* Enqueue the clocksource sorted by rating * Enqueue the clocksource sorted by rating
*/ */
...@@ -693,6 +799,8 @@ static void clocksource_enqueue(struct clocksource *cs) ...@@ -693,6 +799,8 @@ static void clocksource_enqueue(struct clocksource *cs)
entry = &tmp->list; entry = &tmp->list;
} }
list_add(&cs->list, entry); list_add(&cs->list, entry);
ipipe_check_clocksource(cs);
} }
/** /**
......
...@@ -89,7 +89,7 @@ static void tick_periodic(int cpu) ...@@ -89,7 +89,7 @@ static void tick_periodic(int cpu)
update_wall_time(); update_wall_time();
} }
update_process_times(user_mode(get_irq_regs())); update_root_process_times(get_irq_regs());
profile_tick(CPU_PROFILING); profile_tick(CPU_PROFILING);
} }
......
...@@ -158,7 +158,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) ...@@ -158,7 +158,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
ts->next_tick = 0; ts->next_tick = 0;
} }
#endif #endif
update_process_times(user_mode(regs)); update_root_process_times(regs);
profile_tick(CPU_PROFILING); profile_tick(CPU_PROFILING);
} }
#endif #endif
......
...@@ -525,7 +525,7 @@ static inline void update_vsyscall(struct timekeeper *tk) ...@@ -525,7 +525,7 @@ static inline void update_vsyscall(struct timekeeper *tk)
xt = timespec64_to_timespec(tk_xtime(tk)); xt = timespec64_to_timespec(tk_xtime(tk));
wm = timespec64_to_timespec(tk->wall_to_monotonic); wm = timespec64_to_timespec(tk->wall_to_monotonic);
update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult, update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult,
tk->tkr_mono.cycle_last); tk->tkr_mono.shift, tk->tkr_mono.cycle_last);
} }
static inline void old_vsyscall_fixup(struct timekeeper *tk) static inline void old_vsyscall_fixup(struct timekeeper *tk)
......
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
#include <linux/kernel_stat.h> #include <linux/kernel_stat.h>
#include <linux/export.h> #include <linux/export.h>
#include <linux/interrupt.h> #include <linux/interrupt.h>
#include <linux/ipipe.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/mm.h> #include <linux/mm.h>
...@@ -1623,6 +1624,24 @@ static inline void __run_timers(struct timer_base *base) ...@@ -1623,6 +1624,24 @@ static inline void __run_timers(struct timer_base *base)
raw_spin_unlock_irq(&base->lock); raw_spin_unlock_irq(&base->lock);
} }
#ifdef CONFIG_IPIPE
void update_root_process_times(struct pt_regs *regs)
{
int user_tick = user_mode(regs);
if (__ipipe_root_tick_p(regs)) {
update_process_times(user_tick);
return;
}
run_local_timers();
rcu_check_callbacks(user_tick);
run_posix_cpu_timers(current);
}
#endif
/* /*
* This function runs timers and the timer-tq in bottom half context. * This function runs timers and the timer-tq in bottom half context.
*/ */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment