Commit 38dfae7b authored by Gilles Chanteperdrix's avatar Gilles Chanteperdrix Committed by Philippe Gerum

ARM: ipipe: add user-visible TSC helpers

parent bb130ab0
......@@ -912,6 +912,15 @@ config PLAT_PXA
config PLAT_VERSATILE
bool
if IPIPE
config IPIPE_ARM_KUSER_TSC
bool
select HAVE_IPIPE_TRACER_SUPPORT
select GENERIC_TIME_VSYSCALL
select IPIPE_HAVE_HOSTRT if IPIPE
default y if ARM_TIMER_SP804 || ARCH_MXC || ARCH_OMAP
endif
source "arch/arm/firmware/Kconfig"
source arch/arm/mm/Kconfig
......
......@@ -89,6 +89,7 @@ head-y := head$(MMUEXT).o
obj-$(CONFIG_DEBUG_LL) += debug.o
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
obj-$(CONFIG_IPIPE) += ipipe.o
obj-$(CONFIG_IPIPE_ARM_KUSER_TSC) += ipipe_tsc.o ipipe_tsc_asm.o
# This is executed very early using a temporary stack when no memory allocator
# nor global data is available. Everything has to be allocated on the stack.
......
......@@ -907,6 +907,50 @@ ENDPROC(__switch_to)
#endif
.endm
#ifdef CONFIG_IPIPE
/*
I-pipe tsc area, here we store data shared with user-space for
tsc-emulation. If CONFIG_IPIPE_ARM_KUSER_TSC is enabled
__ipipe_kuser_get_tsc will be overwritten with the real TSC
emulation code.
*/
.globl __ipipe_tsc_area
.equ __ipipe_tsc_area, VECTORS_BASE + 0x1000 + __ipipe_tsc_area_start - __kuser_helper_end
#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
.globl __ipipe_tsc_addr
.equ __ipipe_tsc_addr, VECTORS_BASE + 0x1000 + .LCcntr_addr - __kuser_helper_end
.globl __ipipe_tsc_get
.equ __ipipe_tsc_get, VECTORS_BASE + 0x1000 + __ipipe_kuser_get_tsc - __kuser_helper_end
#endif
.align 5
.globl __ipipe_tsc_area_start
__ipipe_tsc_area_start:
.rep 3
.word 0
.endr
#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
.rep 4
.word 0
.endr
.LCcntr_addr:
.word 0
.align 5
__ipipe_kuser_get_tsc:
nop
mov r0, #0
mov r1, #0
usr_ret lr
.rep 20
.word 0
.endr
#endif
#endif
.macro kuser_pad, sym, size
.if (. - \sym) & 3
.rept 4 - (. - \sym) & 3
......
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/clocksource.h>
#include <linux/ipipe_tickdev.h>
#include <linux/cpufreq.h>
#include <linux/ipipe.h>
#include <asm/cacheflush.h>
#include <asm/traps.h>
typedef unsigned long long __ipipe_tsc_t(void);
extern __ipipe_tsc_t __ipipe_freerunning_64,
__ipipe_freerunning_32,
__ipipe_freerunning_countdown_32,
__ipipe_freerunning_16,
__ipipe_freerunning_countdown_16,
__ipipe_decrementer_16,
__ipipe_freerunning_twice_16,
__ipipe_freerunning_arch;
extern unsigned long __ipipe_tsc_addr;
static struct __ipipe_tscinfo tsc_info;
static struct clocksource clksrc = {
.name = "ipipe_tsc",
.rating = 0x7fffffff,
.read = (typeof(clksrc.read))__ipipe_tsc_get,
.mask = CLOCKSOURCE_MASK(64),
.flags = CLOCK_SOURCE_IS_CONTINUOUS,
};
struct ipipe_tsc_value_t {
unsigned long long last_tsc;
unsigned last_cnt;
};
unsigned long __ipipe_kuser_tsc_freq;
struct ipipe_tsc_value_t *ipipe_tsc_value;
static struct ipipe_tsc_update_timer {
struct timer_list timer;
long period;
} ipipe_tsc_update_timer;
static void __ipipe_tsc_update_fn(struct timer_list *t)
{
struct ipipe_tsc_update_timer *upt;
upt = container_of(t, struct ipipe_tsc_update_timer, timer);
__ipipe_tsc_update();
mod_timer(&ipipe_tsc_update_timer.timer, jiffies + upt->period);
}
void __init __ipipe_tsc_register(struct __ipipe_tscinfo *info)
{
struct ipipe_tsc_value_t *vector_tsc_value;
unsigned long long wrap_ms;
unsigned long *tsc_addr;
__ipipe_tsc_t *implem;
unsigned long flags;
int registered;
char *tsc_area;
#if !defined(CONFIG_CPU_USE_DOMAINS)
extern char __ipipe_tsc_area_start[], __kuser_helper_end[];
tsc_area = (char *)vectors_page + 0x1000
+ (__ipipe_tsc_area_start - __kuser_helper_end);
tsc_addr = (unsigned long *)
(tsc_area + ((char *)&__ipipe_tsc_addr - __ipipe_tsc_area));
#else
tsc_area = __ipipe_tsc_area;
tsc_addr = &__ipipe_tsc_addr;
#endif
registered = ipipe_tsc_value != NULL;
if (WARN_ON(info->freq == 0))
return;
if (registered && info->freq < tsc_info.freq)
return;
ipipe_tsc_value = (struct ipipe_tsc_value_t *)tsc_area;
vector_tsc_value = (struct ipipe_tsc_value_t *)__ipipe_tsc_area;
switch(info->type) {
case IPIPE_TSC_TYPE_FREERUNNING:
switch(info->u.mask) {
case 0xffff:
implem = &__ipipe_freerunning_16;
break;
case 0xffffffff:
implem = &__ipipe_freerunning_32;
break;
case 0xffffffffffffffffULL:
implem = &__ipipe_freerunning_64;
break;
default:
goto unimplemented;
}
break;
case IPIPE_TSC_TYPE_DECREMENTER:
if (info->u.mask != 0xffff)
goto unimplemented;
implem = &__ipipe_decrementer_16;
break;
case IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN:
switch(info->u.mask) {
case 0xffff:
implem = &__ipipe_freerunning_countdown_16;
break;
case 0xffffffff:
implem = &__ipipe_freerunning_countdown_32;
break;
default:
goto unimplemented;
}
break;
case IPIPE_TSC_TYPE_FREERUNNING_TWICE:
if (info->u.mask != 0xffff)
goto unimplemented;
implem = &__ipipe_freerunning_twice_16;
break;
case IPIPE_TSC_TYPE_FREERUNNING_ARCH:
implem = &__ipipe_freerunning_arch;
break;
default:
unimplemented:
printk("I-pipe: Unimplemented tsc configuration, "
"type: %d, mask: 0x%08Lx\n", info->type, info->u.mask);
BUG();
}
tsc_info = *info;
*tsc_addr = tsc_info.counter_vaddr;
if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) {
tsc_info.u.dec.last_cnt = &vector_tsc_value->last_cnt;
tsc_info.u.dec.tsc = &vector_tsc_value->last_tsc;
} else
tsc_info.u.fr.tsc = &vector_tsc_value->last_tsc;
flags = hard_local_irq_save();
ipipe_tsc_value->last_tsc = 0;
memcpy(tsc_area + 0x20, implem, 0x60);
flush_icache_range((unsigned long)(tsc_area),
(unsigned long)(tsc_area + 0x80));
hard_local_irq_restore(flags);
__ipipe_kuser_tsc_freq = tsc_info.freq;
wrap_ms = info->u.mask;
do_div(wrap_ms, tsc_info.freq / 1000);
printk(KERN_INFO "I-pipe, %u.%03u MHz clocksource, wrap in %Lu ms\n",
tsc_info.freq / 1000000, (tsc_info.freq % 1000000) / 1000,
wrap_ms);
if (!registered) {
timer_setup(&ipipe_tsc_update_timer.timer, __ipipe_tsc_update_fn, 0);
clocksource_register_hz(&clksrc, tsc_info.freq);
} else
__clocksource_update_freq_hz(&clksrc, tsc_info.freq);
wrap_ms *= HZ / 2;
do_div(wrap_ms, 1000);
if (wrap_ms > 0x7fffffff)
wrap_ms = 0x7fffffff;
ipipe_tsc_update_timer.period = wrap_ms;
mod_timer(&ipipe_tsc_update_timer.timer, jiffies + wrap_ms);
__ipipe_tracer_hrclock_initialized();
}
void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info)
{
*info = tsc_info;
}
void __ipipe_tsc_update(void)
{
if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) {
unsigned cnt = *(unsigned *)tsc_info.counter_vaddr;
int offset = ipipe_tsc_value->last_cnt - cnt;
if (offset < 0)
offset += tsc_info.u.dec.mask + 1;
ipipe_tsc_value->last_tsc += offset;
ipipe_tsc_value->last_cnt = cnt;
return;
}
/* Update last_tsc, in order to remain compatible with legacy
user-space 32 bits free-running counter implementation */
ipipe_tsc_value->last_tsc = __ipipe_tsc_get() - 1;
}
EXPORT_SYMBOL(__ipipe_tsc_get);
void __ipipe_update_vsyscall(struct timekeeper *tk)
{
if (tk->tkr_mono.clock == &clksrc)
ipipe_update_hostrt(tk);
}
#if !IS_ENABLED(CONFIG_VDSO)
void update_vsyscall(struct timekeeper *tk)
{
__ipipe_update_vsyscall(tk);
}
void update_vsyscall_tz(void)
{
}
#endif
#ifdef CONFIG_CPU_FREQ
static __init void update_timer_freq(void *data)
{
unsigned int hrclock_freq = *(unsigned int *)data;
__ipipe_timer_refresh_freq(hrclock_freq);
}
static __init int cpufreq_transition_handler(struct notifier_block *nb,
unsigned long state, void *data)
{
struct cpufreq_freqs *freqs = data;
unsigned int freq;
if (state == CPUFREQ_POSTCHANGE &&
ipipe_tsc_value && tsc_info.refresh_freq) {
freq = tsc_info.refresh_freq();
if (freq) {
if (freqs->cpu == 0) {
int oldrate;
tsc_info.freq = freq;
__ipipe_tsc_register(&tsc_info);
__ipipe_report_clockfreq_update(freq);
/* force timekeeper to recalculate the clocksource */
oldrate = clksrc.rating;
clocksource_change_rating(&clksrc, 0);
clocksource_change_rating(&clksrc, oldrate);
}
smp_call_function_single(freqs->cpu, update_timer_freq,
&freq, 1);
}
}
return NOTIFY_OK;
}
static struct notifier_block __initdata cpufreq_nb = {
.notifier_call = cpufreq_transition_handler,
};
static __init int register_cpufreq_notifier(void)
{
cpufreq_register_notifier(&cpufreq_nb,
CPUFREQ_TRANSITION_NOTIFIER);
return 0;
}
core_initcall(register_cpufreq_notifier);
static __init int unregister_cpufreq_notifier(void)
{
cpufreq_unregister_notifier(&cpufreq_nb,
CPUFREQ_TRANSITION_NOTIFIER);
return 0;
}
late_initcall(unregister_cpufreq_notifier);
#endif /* CONFIG_CPUFREQ */
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
#include <asm/glue.h>
.macro usr_ret, reg
#ifdef CONFIG_ARM_THUMB
bx \reg
#else
mov pc, \reg
#endif
.endm
.macro usr_reteq, reg
#ifdef CONFIG_ARM_THUMB
bxeq \reg
#else
moveq pc, \reg
#endif
.endm
.macro myldrd, rd1, rd2, rtmp, label
#if __LINUX_ARM_ARCH__ < 5
adr \rtmp, \label
ldm \rtmp, { \rd1, \rd2 }
#else
ldrd \rd1, \label
#endif
.endm
/*
We use the same mechanism as Linux user helpers to store
variables and functions related to TSC emulation, so that they
can also be used in user-space.
The function ipipe_tsc_register will copy the proper
implemntation to the vectors page. We repeat the data area so
that the PC relative operations are computed correctly.
*/
.section .init.text, "ax", %progbits
THUMB( .arm )
.align 5
.rep 7
.word 0
.endr
.LCfr64_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_64
__ipipe_freerunning_64:
ldr r0, .LCfr64_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
mov r2, r0
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r1, [r2, #4]
1: ldr r0, [r2]
ldr r3, [r2, #4]
cmp r3, r1
usr_reteq lr
mov r1, r3
b 1b
#else /* Big endian */
ldr r0, [r2]
1: ldr r1, [r2, #4]
ldr r3, [r2]
cmp r3, r0
usr_reteq lr
mov r0, r3
b 1b
#endif /* Big endian */
.align 5
.LCfr32_last_tsc:
.rep 7
.word 0
.endr
.LCfr32_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_32
__ipipe_freerunning_32:
ldr r0, .LCfr32_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
myldrd r2, r3, r1, .LCfr32_last_tsc
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r0, [r0]
cmp r2, r0
adc r1, r3, #0
#else /* Big endian */
ldr r1, [r0]
cmp r3, r1
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.LCfrcd32_last_tsc:
.rep 7
.word 0
.endr
.LCfrcd32_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_countdown_32
__ipipe_freerunning_countdown_32:
ldr r0, .LCfrcd32_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
myldrd r2, r3, r1, .LCfrcd32_last_tsc
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r0, [r0]
mvn r0, r0
cmp r2, r0
adc r1, r3, #0
#else /* Big endian */
ldr r1, [r0]
mvn r1, r1
cmp r3, r1
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.LCfr16_last_tsc:
.rep 7
.word 0
.endr
.LCfr16_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_16
__ipipe_freerunning_16:
ldr r0, .LCfr16_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
1: myldrd r2, r3, r1, .LCfr16_last_tsc
ldrh ip, [r0]
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r1, .LCfr16_last_tsc
cmp r1, r2
mov r1, r2, lsr #16
bne 1b
orr r0, ip, r1, lsl #16
cmp r2, r0
addhis r0, r0, #0x10000
adc r1, r3, #0
#else /* Big endian */
ldr r1, .LCfr16_last_tsc + 4
cmp r1, r3
mov r1, r3, lsr #16
bne 1b
orr r1, ip, r1, lsl #16
cmp r3, r1
addhis r1, r1, #0x10000
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.LCfrcd16_last_tsc:
.rep 7
.word 0
.endr
.LCfrcd16_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_countdown_16
__ipipe_freerunning_countdown_16:
ldr r0, .LCfrcd16_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
1: myldrd r2, r3, r1, .LCfrcd16_last_tsc
ldrh ip, [r0]
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r1, .LCfrcd16_last_tsc
rsb ip, ip, #0x10000
cmp r1, r2
mov r1, r2, lsr #16
bne 1b
orr r0, ip, r1, lsl #16
cmp r2, r0
addhis r0, r0, #0x10000
adc r1, r3, #0
#else /* Big endian */
ldr r1, .LCfrcd16_last_tsc + 4
rsb ip, ip, #0x10000
cmp r1, r3
mov r1, r3, lsr #16
bne 1b
orr r1, ip, r1, lsl #16
cmp r3, r1
addhis r1, r1, #0x10000
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.LCfrt16_last_tsc:
.rep 7
.word 0
.endr
.LCfrt16_cntr_addr:
.word 0
.align 5
.globl __ipipe_freerunning_twice_16
__ipipe_freerunning_twice_16:
ldr r0, .LCfrt16_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
1: myldrd r2, r3, r1, .LCfrt16_last_tsc
2: ldrh ip, [r0]
ldrh r1, [r0]
cmp r1, ip
bne 2b
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
ldr r1, .LCfrt16_last_tsc
cmp r1, r2
mov r1, r2, lsr #16
bne 1b
orr r0, ip, r1, lsl #16
cmp r2, r0
addhis r0, r0, #0x10000
adc r1, r3, #0
#else /* Big endian */
ldr r1, .LCfrt16_last_tsc + 4
cmp r1, r3
mov r1, r3, lsr #16
bne 1b
orr r1, ip, r1, lsl #16
cmp r3, r1
addhis r1, r1, #0x10000
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.LCdec16_last_tsc:
.rep 2
.word 0
.endr
.LCdec16_last_cnt:
.rep 5
.word 0
.endr
.LCdec16_cntr_addr:
.word 0
.align 5
.globl __ipipe_decrementer_16
__ipipe_decrementer_16:
ldr r0, .LCdec16_cntr_addr
/* User-space entry-point: r0 is the hardware counter virtual address */
#ifndef CONFIG_CPU_BIG_ENDIAN
/* Little endian */
1: ldr r1, .LCdec16_last_tsc
ldrh ip, [r0]
ldr r2, .LCdec16_last_cnt
subs ip, r2, ip
addcc ip, ip, #0x10000
myldrd r2, r3, r3, .LCdec16_last_tsc
cmp r1, r2
bne 1b
adds r0, ip, r2
adc r1, r3, #0
#else /* Big endian */
1: ldr r1, .LCdec16_last_tsc + 4
ldrh ip, [r0]
ldr r2, .LCdec16_last_cnt
subs ip, r2, ip
addcc ip, ip, #0x10000
myldrd r2, r3, r3, .LCdec16_last_tsc
cmp r1, r3
bne 1b
adds r1, ip, r3
adc r0, r2, #0
#endif /* Big endian */
usr_ret lr
.align 5
.globl __ipipe_freerunning_arch
__ipipe_freerunning_arch:
nop
#ifdef CONFIG_ARM_ARCH_TIMER
mrrc p15, 0, r0, r1, c14
#else
mov r0, #0
mov r1, #0
#endif
usr_ret lr
......@@ -813,10 +813,21 @@ void __init trap_init(void)
#ifdef CONFIG_KUSER_HELPERS
static void __init kuser_init(void *vectors)
{
#ifndef CONFIG_IPIPE
extern char __kuser_helper_start[], __kuser_helper_end[];
int kuser_sz = __kuser_helper_end - __kuser_helper_start;
#else /* !CONFIG_IPIPE */
extern char __ipipe_tsc_area_start[], __kuser_helper_end[];
int kuser_sz = __kuser_helper_end - __ipipe_tsc_area_start;
extern char __vectors_start[], __vectors_end[];
#endif /* !CONFIG_IPIPE */
#ifndef CONFIG_IPIPE
memcpy(vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz);
#else /* !CONFIG_IPIPE */
BUG_ON(0x1000 - kuser_sz < __vectors_end - __vectors_start);
memcpy(vectors + 0x1000 - kuser_sz, __ipipe_tsc_area_start, kuser_sz);
#endif /* !CONFIG_IPIPE */
/*
* vectors + 0xfe0 = __kuser_get_tls
......
......@@ -21,6 +21,7 @@
#include <linux/elf.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/ipipe.h>
#include <linux/mm.h>
#include <linux/of.h>
#include <linux/printk.h>
......@@ -319,6 +320,8 @@ void update_vsyscall(struct timekeeper *tk)
{
struct timespec64 *wtm = &tk->wall_to_monotonic;
__ipipe_update_vsyscall(tk);
if (!cntvct_ok) {
/* The entry points have been zeroed, so there is no
* point in updating the data page.
......
......@@ -853,6 +853,7 @@ config TLS_REG_EMUL
config NEED_KUSER_HELPERS
bool
default y if IPIPE
config KUSER_HELPERS
bool "Enable kuser helpers in vector page" if !NEED_KUSER_HELPERS
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment