Commit 9d1ea32d authored by Philippe Gerum's avatar Philippe Gerum Committed by Dmitriy Cherkasov

ARM64: ipipe: intercept exceptions

parent 2f89ff0d
...@@ -573,6 +573,13 @@ el1_undef: ...@@ -573,6 +573,13 @@ el1_undef:
/* /*
* Undefined instruction * Undefined instruction
*/ */
#ifdef CONFIG_IPIPE
mov x0, #7 //@ x0 = IPIPE_TRAP_UNDEFINSTR
mov x1, sp //@ x1 = &regs
bl __ipipe_notify_trap //@ branch to trap handler
cmp w0, #0
bne ipipe_fast_svc_irq_exit
#endif /* CONFIG_IPIPE */
enable_dbg enable_dbg
mov x0, sp mov x0, sp
bl do_undefinstr bl do_undefinstr
...@@ -723,7 +730,7 @@ el0_da: ...@@ -723,7 +730,7 @@ el0_da:
mov x1, x25 mov x1, x25
mov x2, sp mov x2, sp
bl do_mem_abort bl do_mem_abort
b ret_to_user b ret_from_exception
el0_ia: el0_ia:
/* /*
* Instruction abort handling * Instruction abort handling
...@@ -736,7 +743,7 @@ el0_ia: ...@@ -736,7 +743,7 @@ el0_ia:
mov x1, x25 mov x1, x25
mov x2, sp mov x2, sp
bl do_mem_abort bl do_mem_abort
b ret_to_user b ret_from_exception
el0_fpsimd_acc: el0_fpsimd_acc:
/* /*
* Floating Point or Advanced SIMD access * Floating Point or Advanced SIMD access
...@@ -769,7 +776,7 @@ el0_sp_pc: ...@@ -769,7 +776,7 @@ el0_sp_pc:
mov x1, x25 mov x1, x25
mov x2, sp mov x2, sp
bl do_sp_pc_abort bl do_sp_pc_abort
b ret_to_user b ret_from_exception
el0_undef: el0_undef:
/* /*
* Undefined instruction * Undefined instruction
...@@ -779,7 +786,7 @@ el0_undef: ...@@ -779,7 +786,7 @@ el0_undef:
ct_user_exit ct_user_exit
mov x0, sp mov x0, sp
bl do_undefinstr bl do_undefinstr
b ret_to_user b ret_from_exception
el0_sys: el0_sys:
/* /*
* System instructions, for trapped cache maintenance instructions * System instructions, for trapped cache maintenance instructions
...@@ -801,7 +808,7 @@ el0_dbg: ...@@ -801,7 +808,7 @@ el0_dbg:
bl do_debug_exception bl do_debug_exception
enable_dbg enable_dbg
ct_user_exit ct_user_exit
b ret_to_user b ret_from_exception
el0_inv: el0_inv:
enable_dbg enable_dbg
ct_user_exit ct_user_exit
...@@ -809,7 +816,7 @@ el0_inv: ...@@ -809,7 +816,7 @@ el0_inv:
mov x1, #BAD_SYNC mov x1, #BAD_SYNC
mov x2, x25 mov x2, x25
bl bad_el0_sync bl bad_el0_sync
b ret_to_user b ret_from_exception
ENDPROC(el0_sync) ENDPROC(el0_sync)
.align 6 .align 6
......
...@@ -184,10 +184,12 @@ static void ptrace_hbptriggered(struct perf_event *bp, ...@@ -184,10 +184,12 @@ static void ptrace_hbptriggered(struct perf_event *bp,
.si_code = TRAP_HWBKPT, .si_code = TRAP_HWBKPT,
.si_addr = (void __user *)(bkpt->trigger), .si_addr = (void __user *)(bkpt->trigger),
}; };
int i __maybe_unused;
#ifdef CONFIG_COMPAT if (__ipipe_report_trap(IPIPE_TRAP_BREAK, regs))
int i; return;
#ifdef CONFIG_COMPAT
if (!is_compat_task()) if (!is_compat_task())
goto send_sig; goto send_sig;
......
...@@ -642,7 +642,12 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) ...@@ -642,7 +642,12 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
esr_get_class_string(esr)); esr_get_class_string(esr));
die("Oops - bad mode", regs, 0); die("Oops - bad mode", regs, 0);
#ifdef CONFIG_IPIPE
hard_local_irq_disable();
__ipipe_root_status &= ~IPIPE_STALL_FLAG;
#else
local_irq_disable(); local_irq_disable();
#endif
panic("bad mode"); panic("bad mode");
} }
...@@ -654,6 +659,15 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) ...@@ -654,6 +659,15 @@ asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
{ {
siginfo_t info; siginfo_t info;
void __user *pc = (void __user *)instruction_pointer(regs); void __user *pc = (void __user *)instruction_pointer(regs);
if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN,regs))
return;
#ifdef CONFIG_IPIPE
ipipe_stall_root();
hard_local_irq_enable();
#endif
console_verbose(); console_verbose();
pr_crit("Bad EL0 synchronous exception detected on CPU%d, code 0x%08x -- %s\n", pr_crit("Bad EL0 synchronous exception detected on CPU%d, code 0x%08x -- %s\n",
......
...@@ -61,6 +61,39 @@ static inline const struct fault_info *esr_to_fault_info(unsigned int esr) ...@@ -61,6 +61,39 @@ static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
return fault_info + (esr & 63); return fault_info + (esr & 63);
} }
#ifdef CONFIG_IPIPE
static inline unsigned long ipipe_fault_entry(void)
{
unsigned long flags;
int s;
flags = hard_local_irq_save();
s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
hard_local_irq_enable();
return arch_mangle_irq_bits(s, flags);
}
static inline void ipipe_fault_exit(unsigned long x)
{
if (!arch_demangle_irq_bits(&x))
local_irq_enable();
else
hard_local_irq_restore(x);
}
#else
static inline unsigned long ipipe_fault_entry(void)
{
return 0;
}
static inline void ipipe_fault_exit(unsigned long x) { }
#endif
#ifdef CONFIG_KPROBES #ifdef CONFIG_KPROBES
static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr) static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
{ {
...@@ -126,6 +159,16 @@ static void mem_abort_decode(unsigned int esr) ...@@ -126,6 +159,16 @@ static void mem_abort_decode(unsigned int esr)
data_abort_decode(esr); data_abort_decode(esr);
} }
#define cpu_get_pgd() \
({ \
unsigned long pg; \
asm("mrs %0, ttbr0_el1\n" \
: "=r" (pg)); \
pg &= ~0xffff000000003ffful; \
(pgd_t *)phys_to_virt(pg); \
})
/* /*
* Dump out the page tables associated with 'addr' in the currently active mm. * Dump out the page tables associated with 'addr' in the currently active mm.
*/ */
...@@ -151,9 +194,9 @@ void show_pte(unsigned long addr) ...@@ -151,9 +194,9 @@ void show_pte(unsigned long addr)
return; return;
} }
pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgd = %p\n", pr_alert("%s pgtable: %luk pages, %u-bit VAs, mm_pgd = %p, hw_pgd = %p\n",
mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
VA_BITS, mm->pgd); VA_BITS, mm->pgd, cpu_get_pgd());
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd)); pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd));
...@@ -256,12 +299,21 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, ...@@ -256,12 +299,21 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr,
struct pt_regs *regs) struct pt_regs *regs)
{ {
const char *msg; const char *msg;
unsigned long flags;
int ret;
/* /*
* Are we prepared to handle this kernel fault? * Are we prepared to handle this kernel fault?
* We are almost certainly not prepared to handle instruction faults. * We are almost certainly not prepared to handle instruction faults.
*/ */
if (!is_el1_instruction_abort(esr) && fixup_exception(regs)) flags = hard_cond_local_irq_save();
ret = (!is_el1_instruction_abort(esr) && fixup_exception(regs));
hard_cond_local_irq_restore(flags);
if (ret)
return;
if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs))
return; return;
/* /*
...@@ -302,6 +354,12 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr, ...@@ -302,6 +354,12 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
struct siginfo si; struct siginfo si;
const struct fault_info *inf; const struct fault_info *inf;
unsigned int lsb = 0; unsigned int lsb = 0;
unsigned long irqflags;
if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs))
return;
irqflags = ipipe_fault_entry();
if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) {
inf = esr_to_fault_info(esr); inf = esr_to_fault_info(esr);
...@@ -331,6 +389,8 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr, ...@@ -331,6 +389,8 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr,
si.si_addr_lsb = lsb; si.si_addr_lsb = lsb;
force_sig_info(sig, &si, tsk); force_sig_info(sig, &si, tsk);
ipipe_fault_exit(irqflags);
} }
static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
...@@ -402,10 +462,16 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -402,10 +462,16 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
int fault, sig, code, major = 0; int fault, sig, code, major = 0;
unsigned long vm_flags = VM_READ | VM_WRITE; unsigned long vm_flags = VM_READ | VM_WRITE;
unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
unsigned long irqflags;
if (notify_page_fault(regs, esr)) if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs))
return 0; return 0;
irqflags = ipipe_fault_entry();
if (notify_page_fault(regs, esr))
goto out;
tsk = current; tsk = current;
mm = tsk->mm; mm = tsk->mm;
...@@ -475,7 +541,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -475,7 +541,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
if (fatal_signal_pending(current)) { if (fatal_signal_pending(current)) {
if (!user_mode(regs)) if (!user_mode(regs))
goto no_context; goto no_context;
return 0; goto out;
} }
/* /*
...@@ -511,7 +577,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -511,7 +577,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
addr); addr);
} }
return 0; goto out;
} }
/* /*
...@@ -528,7 +594,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -528,7 +594,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
* oom-killed). * oom-killed).
*/ */
pagefault_out_of_memory(); pagefault_out_of_memory();
return 0; goto out;
} }
if (fault & VM_FAULT_SIGBUS) { if (fault & VM_FAULT_SIGBUS) {
...@@ -552,10 +618,14 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, ...@@ -552,10 +618,14 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
} }
__do_user_fault(tsk, addr, esr, sig, code, regs, fault); __do_user_fault(tsk, addr, esr, sig, code, regs, fault);
return 0; goto out;
no_context: no_context:
__do_kernel_fault(addr, esr, regs); __do_kernel_fault(addr, esr, regs);
out:
ipipe_fault_exit(irqflags);
return 0; return 0;
} }
...@@ -584,6 +654,7 @@ static int __kprobes do_translation_fault(unsigned long addr, ...@@ -584,6 +654,7 @@ static int __kprobes do_translation_fault(unsigned long addr,
return do_page_fault(addr, esr, regs); return do_page_fault(addr, esr, regs);
do_bad_area(addr, esr, regs); do_bad_area(addr, esr, regs);
return 0; return 0;
} }
...@@ -599,6 +670,9 @@ static int do_alignment_fault(unsigned long addr, unsigned int esr, ...@@ -599,6 +670,9 @@ static int do_alignment_fault(unsigned long addr, unsigned int esr,
*/ */
static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs) static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
{ {
if (__ipipe_report_trap(IPIPE_TRAP_DABT, regs))
return 0;
return 1; return 1;
} }
...@@ -736,9 +810,14 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, ...@@ -736,9 +810,14 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
const struct fault_info *inf = esr_to_fault_info(esr); const struct fault_info *inf = esr_to_fault_info(esr);
struct siginfo info; struct siginfo info;
IPIPE_WARN_ONCE(hard_irqs_disabled());
if (!inf->fn(addr, esr, regs)) if (!inf->fn(addr, esr, regs))
return; return;
if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN, regs))
return;
pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n", pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
inf->name, esr, addr); inf->name, esr, addr);
...@@ -767,6 +846,9 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, ...@@ -767,6 +846,9 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
esr_get_class_string(esr), (void *)regs->pc, esr_get_class_string(esr), (void *)regs->pc,
(void *)regs->sp); (void *)regs->sp);
if (__ipipe_report_trap(IPIPE_TRAP_ALIGNMENT, regs))
return;
info.si_signo = SIGBUS; info.si_signo = SIGBUS;
info.si_errno = 0; info.si_errno = 0;
info.si_code = BUS_ADRALN; info.si_code = BUS_ADRALN;
...@@ -810,6 +892,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, ...@@ -810,6 +892,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
struct pt_regs *regs) struct pt_regs *regs)
{ {
const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr); const struct fault_info *inf = debug_fault_info + DBG_ESR_EVT(esr);
unsigned long irqflags;
struct siginfo info; struct siginfo info;
int rv; int rv;
...@@ -823,6 +906,11 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, ...@@ -823,6 +906,11 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
if (!inf->fn(addr, esr, regs)) { if (!inf->fn(addr, esr, regs)) {
rv = 1; rv = 1;
} else { } else {
if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN, regs))
return 0;
irqflags = ipipe_fault_entry();
pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n",
inf->name, esr, addr); inf->name, esr, addr);
...@@ -832,6 +920,8 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, ...@@ -832,6 +920,8 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
info.si_addr = (void __user *)addr; info.si_addr = (void __user *)addr;
arm64_notify_die("", regs, &info, 0); arm64_notify_die("", regs, &info, 0);
rv = 0; rv = 0;
ipipe_fault_exit(irqflags);
} }
if (interrupts_enabled(regs)) if (interrupts_enabled(regs))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment