Commit 0e05bbe2 authored by Philippe Gerum's avatar Philippe Gerum

ftrace: ipipe: enable tracing from the head domain

Enabling ftrace for a co-kernel running in the head domain of a
pipelined interrupt context means to:

- make sure that ftrace's live kernel code patching still runs
  unpreempted by any head domain activity (so that the latter can't
  tread on invalid or half-baked changes in the .text section).

- allow the co-kernel code running in the head domain to traverse
  ftrace's tracepoints safely.

The changes introduced by this commit ensure this by fixing up some
key critical sections so that interrupts are still disabled in the
CPU, undoing the interrupt flag virtualization in those particular
cases.
parent 466697cd
......@@ -160,6 +160,7 @@ enum {
FTRACE_OPS_FL_PID = 1 << 13,
FTRACE_OPS_FL_RCU = 1 << 14,
FTRACE_OPS_FL_TRACE_ARRAY = 1 << 15,
FTRACE_OPS_FL_IPIPE_EXCLUSIVE = 1 << 17,
};
#ifdef CONFIG_DYNAMIC_FTRACE
......
......@@ -525,6 +525,7 @@ config DYNAMIC_FTRACE
bool "enable/disable function tracing dynamically"
depends on FUNCTION_TRACER
depends on HAVE_DYNAMIC_FTRACE
depends on !IPIPE
default y
help
This option will modify all the calls to function tracing
......
......@@ -34,6 +34,7 @@
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/rcupdate.h>
#include <linux/ipipe.h>
#include <trace/events/sched.h>
......@@ -213,8 +214,17 @@ static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
static void update_ftrace_function(void)
{
struct ftrace_ops *ops;
ftrace_func_t func;
for (ops = ftrace_ops_list;
ops != &ftrace_list_end; ops = ops->next)
if (ops->flags & FTRACE_OPS_FL_IPIPE_EXCLUSIVE) {
set_function_trace_op = ops;
func = ops->func;
goto set_pointers;
}
/*
* Prepare the ftrace_ops that the arch callback will use.
* If there's only one ftrace_ops registered, the ftrace_ops_list
......@@ -244,6 +254,7 @@ static void update_ftrace_function(void)
update_function_graph_func();
set_pointers:
/* If there's no change, then do nothing more here */
if (ftrace_trace_function == func)
return;
......@@ -2624,6 +2635,9 @@ void __weak arch_ftrace_update_code(int command)
static void ftrace_run_update_code(int command)
{
#ifdef CONFIG_IPIPE
unsigned long flags;
#endif /* CONFIG_IPIPE */
int ret;
ret = ftrace_arch_code_modify_prepare();
......@@ -2637,7 +2651,13 @@ static void ftrace_run_update_code(int command)
* is safe. The stop_machine() is the safest, but also
* produces the most overhead.
*/
#ifdef CONFIG_IPIPE
flags = ipipe_critical_enter(NULL);
__ftrace_modify_code(&command);
ipipe_critical_exit(flags);
#else /* !CONFIG_IPIPE */
arch_ftrace_update_code(command);
#endif /* !CONFIG_IPIPE */
ret = ftrace_arch_code_modify_post_process();
FTRACE_WARN_ON(ret);
......@@ -5587,10 +5607,10 @@ static int ftrace_process_locs(struct module *mod,
* reason to cause large interrupt latencies while we do it.
*/
if (!mod)
local_irq_save(flags);
flags = hard_local_irq_save();
ftrace_update_code(mod, start_pg);
if (!mod)
local_irq_restore(flags);
hard_local_irq_restore(flags);
ret = 0;
out:
mutex_unlock(&ftrace_lock);
......@@ -6132,9 +6152,11 @@ void __init ftrace_init(void)
unsigned long count, flags;
int ret;
local_irq_save(flags);
flags = hard_local_irq_save_notrace();
ret = ftrace_dyn_arch_init();
local_irq_restore(flags);
hard_local_irq_restore_notrace(flags);
/* ftrace_dyn_arch_init places the return code in addr */
if (ret)
goto failed;
......@@ -6287,7 +6309,16 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
}
} while_for_each_ftrace_op(op);
out:
preempt_enable_notrace();
#ifdef CONFIG_IPIPE
if (hard_irqs_disabled() || !__ipipe_root_p)
/*
* Nothing urgent to schedule here. At latest the timer tick
* will pick up whatever the tracing functions kicked off.
*/
preempt_enable_no_resched_notrace();
else
#endif
preempt_enable_notrace();
trace_clear_recursion(bit);
}
......
......@@ -2653,6 +2653,7 @@ trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
{
unsigned int val = cpu_buffer->current_context;
unsigned long pc = preempt_count();
unsigned long flags;
int bit;
if (!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
......@@ -2661,20 +2662,30 @@ trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
bit = pc & NMI_MASK ? RB_CTX_NMI :
pc & HARDIRQ_MASK ? RB_CTX_IRQ : RB_CTX_SOFTIRQ;
if (unlikely(val & (1 << (bit + cpu_buffer->nest))))
flags = hard_local_irq_save();
if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) {
hard_local_irq_restore(flags);
return 1;
}
val |= (1 << (bit + cpu_buffer->nest));
cpu_buffer->current_context = val;
hard_local_irq_restore(flags);
return 0;
}
static __always_inline void
trace_recursive_unlock(struct ring_buffer_per_cpu *cpu_buffer)
{
unsigned long flags;
flags = hard_local_irq_save();
cpu_buffer->current_context &=
cpu_buffer->current_context - (1 << cpu_buffer->nest);
hard_local_irq_restore(flags);
}
/* The recursive locking above uses 4 bits */
......
......@@ -2918,8 +2918,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
/* Don't pollute graph traces with trace_vprintk internals */
pause_graph_tracing();
flags = hard_local_irq_save();
pc = preempt_count();
preempt_disable_notrace();
tbuffer = get_trace_buf();
if (!tbuffer) {
......@@ -2932,7 +2933,6 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
goto out;
local_save_flags(flags);
size = sizeof(*entry) + sizeof(u32) * len;
buffer = tr->trace_buffer.buffer;
event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
......@@ -2953,7 +2953,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
put_trace_buf();
out_nobuffer:
preempt_enable_notrace();
hard_local_irq_restore(flags);
unpause_graph_tracing();
return len;
......
......@@ -97,7 +97,7 @@ u64 notrace trace_clock_global(void)
int this_cpu;
u64 now;
raw_local_irq_save(flags);
flags = hard_local_irq_save_notrace();
this_cpu = raw_smp_processor_id();
now = sched_clock_cpu(this_cpu);
......@@ -123,7 +123,7 @@ u64 notrace trace_clock_global(void)
arch_spin_unlock(&trace_clock_struct.lock);
out:
raw_local_irq_restore(flags);
hard_local_irq_restore_notrace(flags);
return now;
}
......
......@@ -190,7 +190,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
* Need to use raw, since this must be called before the
* recursive protection is performed.
*/
local_irq_save(flags);
flags = hard_local_irq_save();
cpu = raw_smp_processor_id();
data = per_cpu_ptr(tr->trace_buffer.data, cpu);
disabled = atomic_inc_return(&data->disabled);
......@@ -202,7 +202,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
}
atomic_dec(&data->disabled);
local_irq_restore(flags);
hard_local_irq_restore(flags);
}
static struct tracer_opt func_opts[] = {
......
......@@ -435,7 +435,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
if (tracing_thresh)
return 1;
local_irq_save(flags);
flags = hard_local_irq_save_notrace();
cpu = raw_smp_processor_id();
data = per_cpu_ptr(tr->trace_buffer.data, cpu);
disabled = atomic_inc_return(&data->disabled);
......@@ -447,7 +447,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
}
atomic_dec(&data->disabled);
local_irq_restore(flags);
hard_local_irq_restore_notrace(flags);
return ret;
}
......@@ -511,7 +511,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
ftrace_graph_addr_finish(trace);
local_irq_save(flags);
flags = hard_local_irq_save_notrace();
cpu = raw_smp_processor_id();
data = per_cpu_ptr(tr->trace_buffer.data, cpu);
disabled = atomic_inc_return(&data->disabled);
......@@ -520,7 +520,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace)
__trace_graph_return(tr, trace, flags, pc);
}
atomic_dec(&data->disabled);
local_irq_restore(flags);
hard_local_irq_restore_notrace(flags);
}
void set_graph_array(struct trace_array *tr)
......
......@@ -20,7 +20,7 @@ static DEFINE_PER_CPU(int, tracing_irq_cpu);
void trace_hardirqs_on(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
if (ipipe_root_p && this_cpu_read(tracing_irq_cpu)) {
if (!in_nmi())
trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
......@@ -33,7 +33,7 @@ EXPORT_SYMBOL(trace_hardirqs_on);
void trace_hardirqs_off(void)
{
if (!this_cpu_read(tracing_irq_cpu)) {
if (ipipe_root_p && !this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
if (!in_nmi())
......@@ -46,7 +46,7 @@ EXPORT_SYMBOL(trace_hardirqs_off);
__visible void trace_hardirqs_on_caller(unsigned long caller_addr)
{
if (this_cpu_read(tracing_irq_cpu)) {
if (ipipe_root_p && this_cpu_read(tracing_irq_cpu)) {
if (!in_nmi())
trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
......@@ -59,7 +59,7 @@ EXPORT_SYMBOL(trace_hardirqs_on_caller);
__visible void trace_hardirqs_off_caller(unsigned long caller_addr)
{
if (!this_cpu_read(tracing_irq_cpu)) {
if (ipipe_root_p && !this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
if (!in_nmi())
......@@ -75,14 +75,14 @@ EXPORT_SYMBOL(trace_hardirqs_off_caller);
void trace_preempt_on(unsigned long a0, unsigned long a1)
{
if (!in_nmi())
if (ipipe_root_p && !in_nmi())
trace_preempt_enable_rcuidle(a0, a1);
tracer_preempt_on(a0, a1);
}
void trace_preempt_off(unsigned long a0, unsigned long a1)
{
if (!in_nmi())
if (ipipe_root_p && !in_nmi())
trace_preempt_disable_rcuidle(a0, a1);
tracer_preempt_off(a0, a1);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment