...
 
Commits (37)
......@@ -123,7 +123,7 @@ config X86
select HAVE_CC_STACKPROTECTOR
select HAVE_CMPXCHG_DOUBLE
select HAVE_CMPXCHG_LOCAL
select HAVE_CONTEXT_TRACKING if X86_64
select HAVE_CONTEXT_TRACKING if X86_64 && !IPIPE
select HAVE_COPY_THREAD_TLS
select HAVE_C_RECORDMCOUNT
select HAVE_DEBUG_KMEMLEAK
......@@ -145,6 +145,11 @@ config X86
select HAVE_IOREMAP_PROT
select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64
select HAVE_IRQ_TIME_ACCOUNTING
select HAVE_IPIPE_SUPPORT if X86_64
select HAVE_IPIPE_TRACER_SUPPORT
select IPIPE_HAVE_SAFE_THREAD_INFO if IPIPE
select IPIPE_WANT_PTE_PINNING if IPIPE
select IPIPE_HAVE_VM_NOTIFIER if IPIPE
select HAVE_KERNEL_BZIP2
select HAVE_KERNEL_GZIP
select HAVE_KERNEL_LZ4
......@@ -736,6 +741,7 @@ if HYPERVISOR_GUEST
config PARAVIRT
bool "Enable paravirtualization code"
depends on !IPIPE
---help---
This changes the kernel so it can modify itself when it is run
under a hypervisor, potentially improving performance significantly
......@@ -989,6 +995,8 @@ config SCHED_MC_PRIO
source "kernel/Kconfig.preempt"
source "kernel/ipipe/Kconfig"
config UP_LATE_INIT
def_bool y
depends on !SMP && X86_LOCAL_APIC
......
......@@ -17,6 +17,7 @@
#include <linux/tracehook.h>
#include <linux/audit.h>
#include <linux/seccomp.h>
#include <linux/unistd.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <linux/context_tracking.h>
......@@ -46,6 +47,22 @@ __visible inline void enter_from_user_mode(void)
static inline void enter_from_user_mode(void) {}
#endif
#ifdef CONFIG_IPIPE
#define disable_local_irqs() do { \
hard_local_irq_disable(); \
trace_hardirqs_off(); \
} while (0)
#define enable_local_irqs() do { \
trace_hardirqs_on(); \
hard_local_irq_enable(); \
} while (0)
#define check_irqs_disabled() hard_irqs_disabled()
#else
#define disable_local_irqs() local_irq_disable()
#define enable_local_irqs() local_irq_enable()
#define check_irqs_disabled() irqs_disabled()
#endif
static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
{
#ifdef CONFIG_X86_64
......@@ -146,7 +163,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
*/
while (true) {
/* We have work to do. */
local_irq_enable();
enable_local_irqs();
if (cached_flags & _TIF_NEED_RESCHED)
schedule();
......@@ -170,7 +187,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
klp_update_patch_state(current);
/* Disable IRQs and retry */
local_irq_disable();
disable_local_irqs();
cached_flags = READ_ONCE(current_thread_info()->flags);
......@@ -187,8 +204,8 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
addr_limit_user_check();
if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
local_irq_disable();
if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!check_irqs_disabled()))
disable_local_irqs();
lockdep_sys_exit();
......@@ -253,17 +270,19 @@ __visible inline void syscall_return_slowpath(struct pt_regs *regs)
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
local_irq_enable();
WARN(check_irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
enable_local_irqs();
/*
* First do one-time work. If these work items are enabled, we
* want to run them exactly once per syscall exit with IRQs on.
*/
if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
if (unlikely((!IS_ENABLED(CONFIG_IPIPE) ||
syscall_get_nr(current, regs) < NR_syscalls) &&
(cached_flags & SYSCALL_EXIT_WORK_FLAGS)))
syscall_slow_exit_work(regs, cached_flags);
local_irq_disable();
disable_local_irqs();
prepare_exit_to_usermode(regs);
}
......@@ -272,9 +291,18 @@ __visible void do_syscall_64(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
unsigned long nr = regs->orig_ax;
int ret;
enter_from_user_mode();
local_irq_enable();
enable_local_irqs();
ret = ipipe_handle_syscall(ti, nr & __SYSCALL_MASK, regs);
if (ret > 0) {
disable_local_irqs();
return;
}
if (ret < 0)
goto done;
if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
nr = syscall_trace_enter(regs);
......@@ -290,12 +318,45 @@ __visible void do_syscall_64(struct pt_regs *regs)
regs->di, regs->si, regs->dx,
regs->r10, regs->r8, regs->r9);
}
done:
syscall_return_slowpath(regs);
}
#endif
#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
#ifdef CONFIG_IPIPE
#ifdef CONFIG_X86_32
static inline int pipeline_syscall(struct thread_info *ti,
unsigned long nr, struct pt_regs *regs)
{
return ipipe_handle_syscall(ti, nr, regs);
}
#else
static inline int pipeline_syscall(struct thread_info *ti,
unsigned long nr, struct pt_regs *regs)
{
struct pt_regs regs64 = *regs;
int ret;
regs64.di = (unsigned int)regs->bx;
regs64.si = (unsigned int)regs->cx;
regs64.r10 = (unsigned int)regs->si;
regs64.r8 = (unsigned int)regs->di;
regs64.r9 = (unsigned int)regs->bp;
ret = ipipe_handle_syscall(ti, nr, &regs64);
regs->ax = (unsigned int)regs64.ax;
return ret;
}
#endif /* CONFIG_X86_32 */
#else /* CONFIG_IPIPE */
static inline int pipeline_syscall(struct thread_info *ti,
unsigned long nr, struct pt_regs *regs)
{
return 0;
}
#endif /* CONFIG_IPIPE */
/*
* Does a 32-bit syscall. Called with IRQs on in CONTEXT_KERNEL. Does
* all entry and exit work and returns with IRQs off. This function is
......@@ -306,11 +367,20 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
{
struct thread_info *ti = current_thread_info();
unsigned int nr = (unsigned int)regs->orig_ax;
int ret;
#ifdef CONFIG_IA32_EMULATION
ti->status |= TS_COMPAT;
#endif
ret = pipeline_syscall(ti, nr, regs);
if (ret > 0) {
disable_local_irqs();
return;
}
if (ret < 0)
goto done;
if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
/*
* Subtlety here: if ptrace pokes something larger than
......@@ -334,7 +404,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
(unsigned int)regs->dx, (unsigned int)regs->si,
(unsigned int)regs->di, (unsigned int)regs->bp);
}
done:
syscall_return_slowpath(regs);
}
......@@ -342,7 +412,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
__visible void do_int80_syscall_32(struct pt_regs *regs)
{
enter_from_user_mode();
local_irq_enable();
enable_local_irqs();
do_syscall_32_irqs_on(regs);
}
......@@ -366,7 +436,7 @@ __visible long do_fast_syscall_32(struct pt_regs *regs)
enter_from_user_mode();
local_irq_enable();
enable_local_irqs();
/* Fetch EBP from where the vDSO stashed it. */
if (
......@@ -384,7 +454,7 @@ __visible long do_fast_syscall_32(struct pt_regs *regs)
) {
/* User code screwed up. */
local_irq_disable();
disable_local_irqs();
regs->ax = -EFAULT;
prepare_exit_to_usermode(regs);
return 0; /* Keep it simple: use IRET. */
......
......@@ -30,6 +30,7 @@
#include <asm/hw_irq.h>
#include <asm/page_types.h>
#include <asm/irqflags.h>
#include <asm/ipipe_base.h>
#include <asm/paravirt.h>
#include <asm/percpu.h>
#include <asm/asm.h>
......@@ -57,7 +58,7 @@ END(native_usergs_sysret64)
#ifdef CONFIG_TRACE_IRQFLAGS
bt $9, EFLAGS(%rsp) /* interrupts off? */
jnc 1f
TRACE_IRQS_ON
TRACE_IRQS_ON_VIRT
1:
#endif
.endm
......@@ -73,7 +74,8 @@ END(native_usergs_sysret64)
* make sure the stack pointer does not get reset back to the top
* of the debug stack, and instead just reuses the current stack.
*/
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS)
#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS) \
&& !defined(CONFIG_IPIPE)
.macro TRACE_IRQS_OFF_DEBUG
call debug_stack_set_zero
......@@ -383,6 +385,7 @@ END(__switch_to_asm)
*/
ENTRY(ret_from_fork)
UNWIND_HINT_EMPTY
HARD_COND_ENABLE_INTERRUPTS
movq %rax, %rdi
call schedule_tail /* rdi: 'prev' task parameter */
......@@ -554,8 +557,13 @@ END(irq_entries_start)
1:
ENTER_IRQ_STACK old_rsp=%rdi
/* We entered an interrupt context - irqs are off: */
#ifndef CONFIG_IPIPE
/* We entered an interrupt context - irqs are off unless
pipelining is enabled, in which case we defer tracing until
__ipipe_do_sync_stage() where the virtual IRQ state is
updated for the root stage. */
TRACE_IRQS_OFF
#endif
call \func /* rdi points to pt_regs */
.endm
......@@ -568,7 +576,17 @@ END(irq_entries_start)
common_interrupt:
ASM_CLAC
addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */
#ifdef CONFIG_IPIPE
interrupt __ipipe_handle_irq
testl %eax, %eax
jnz ret_from_intr
LEAVE_IRQ_STACK
testb $3, CS(%rsp)
jz retint_kernel_early
jmp retint_user_early
#else
interrupt do_IRQ
#endif
/* 0(%rsp): old RSP */
ret_from_intr:
DISABLE_INTERRUPTS(CLBR_ANY)
......@@ -583,6 +601,7 @@ ret_from_intr:
GLOBAL(retint_user)
mov %rsp,%rdi
call prepare_exit_to_usermode
retint_user_early:
TRACE_IRQS_IRETQ
GLOBAL(swapgs_restore_regs_and_return_to_usermode)
......@@ -634,13 +653,18 @@ retint_kernel:
jnc 1f
0: cmpl $0, PER_CPU_VAR(__preempt_count)
jnz 1f
#ifdef CONFIG_IPIPE
call __ipipe_preempt_schedule_irq
#else
call preempt_schedule_irq
#endif
jmp 0b
1:
#endif
/*
* The iretq could re-enable interrupts:
*/
*/
retint_kernel_early:
TRACE_IRQS_IRETQ
GLOBAL(restore_regs_and_return_to_kernel)
......@@ -754,6 +778,26 @@ END(common_interrupt)
/*
* APIC interrupts.
*/
#ifdef CONFIG_IPIPE
.macro apicinterrupt2 num sym
ENTRY(\sym)
UNWIND_HINT_IRET_REGS
ASM_CLAC
pushq $~(\num)
.Lcommon_\sym:
interrupt __ipipe_handle_irq
testl %eax, %eax
jnz ret_from_intr
LEAVE_IRQ_STACK
testb $3, CS(%rsp)
jz retint_kernel_early
jmp retint_user_early
END(\sym)
.endm
.macro apicinterrupt3 num sym do_sym
apicinterrupt2 \num \sym
.endm
#else /* !CONFIG_IPIPE */
.macro apicinterrupt3 num sym do_sym
ENTRY(\sym)
UNWIND_HINT_IRET_REGS
......@@ -764,6 +808,7 @@ ENTRY(\sym)
jmp ret_from_intr
END(\sym)
.endm
#endif /* !CONFIG_IPIPE */
/* Make sure APIC interrupt handlers end up in the irqentry section: */
#define PUSH_SECTION_IRQENTRY .pushsection .irqentry.text, "ax"
......@@ -809,6 +854,14 @@ apicinterrupt THERMAL_APIC_VECTOR thermal_interrupt smp_thermal_interrupt
apicinterrupt CALL_FUNCTION_SINGLE_VECTOR call_function_single_interrupt smp_call_function_single_interrupt
apicinterrupt CALL_FUNCTION_VECTOR call_function_interrupt smp_call_function_interrupt
apicinterrupt RESCHEDULE_VECTOR reschedule_interrupt smp_reschedule_interrupt
#ifdef CONFIG_IPIPE
apicinterrupt2 IPIPE_RESCHEDULE_VECTOR ipipe_reschedule_interrupt
apicinterrupt2 IPIPE_CRITICAL_VECTOR ipipe_critical_interrupt
#endif
#endif
#ifdef CONFIG_IPIPE
apicinterrupt2 IPIPE_HRTIMER_VECTOR ipipe_hrtimer_interrupt
#endif
apicinterrupt ERROR_APIC_VECTOR error_interrupt smp_error_interrupt
......@@ -851,7 +904,47 @@ ENTRY(switch_to_thread_stack)
ret
END(switch_to_thread_stack)
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
.macro ipipe_idtentry_prologue paranoid=0 trapnr=-1 skip_label=-invalid-
#ifdef CONFIG_IPIPE
movq EFLAGS(%rsp), %r14 /* regs->flags */
movq %rsp, %rdi /* pt_regs pointer */
movl $\trapnr, %esi /* trap number */
subq $8, %rsp
movq %rsp, %rdx /* &flags */
call __ipipe_trap_prologue
popq %r13
mov %rax, %r12 /* save propagation status */
.if \paranoid == 0 /* paranoid may not skip handler */
testl %eax, %eax
jg \skip_label /* skip regular handler if > 0 */
.endif
#endif
.endm
.macro ipipe_idtentry_epilogue paranoid=0 skip_label=-invalid-
#ifdef CONFIG_IPIPE
testl %r12d, %r12d
jnz 1000f
movq %rsp, %rdi /* pt_regs pointer */
movq %r13, %rsi /* &flags from prologue */
movq %r14, %rdx /* original regs->flags before fixup */
call __ipipe_trap_epilogue
1000:
.if \paranoid == 0 /* paranoid implies normal epilogue */
testl %r12d, %r12d
jz 1001f
\skip_label:
UNWIND_HINT_REGS
DISABLE_INTERRUPTS(CLBR_ANY)
testl %ebx, %ebx /* %ebx: return to kernel mode */
jnz retint_kernel_early
jmp retint_user_early
.endif
1001:
#endif
.endm
.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 trapnr=-1
ENTRY(\sym)
UNWIND_HINT_IRET_REGS offset=\has_error_code*8
......@@ -887,6 +980,8 @@ ENTRY(\sym)
.endif
.endif
ipipe_idtentry_prologue paranoid=\paranoid trapnr=\trapnr skip_label=kernel_skip_\@
movq %rsp, %rdi /* pt_regs pointer */
.if \has_error_code
......@@ -902,6 +997,8 @@ ENTRY(\sym)
call \do_sym
ipipe_idtentry_epilogue paranoid=\paranoid skip_label=kernel_skip_\@
.if \shift_ist != -1
addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist)
.endif
......@@ -922,6 +1019,8 @@ ENTRY(\sym)
.Lfrom_usermode_switch_stack_\@:
call error_entry
ipipe_idtentry_prologue paranoid=\paranoid trapnr=\trapnr skip_label=user_skip_\@
movq %rsp, %rdi /* pt_regs pointer */
.if \has_error_code
......@@ -933,24 +1032,26 @@ ENTRY(\sym)
call \do_sym
ipipe_idtentry_epilogue paranoid=\paranoid skip_label=user_skip_\@
jmp error_exit
.endif
END(\sym)
.endm
idtentry divide_error do_divide_error has_error_code=0
idtentry overflow do_overflow has_error_code=0
idtentry bounds do_bounds has_error_code=0
idtentry invalid_op do_invalid_op has_error_code=0
idtentry device_not_available do_device_not_available has_error_code=0
idtentry double_fault do_double_fault has_error_code=1 paranoid=2
idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
idtentry invalid_TSS do_invalid_TSS has_error_code=1
idtentry segment_not_present do_segment_not_present has_error_code=1
idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0
idtentry coprocessor_error do_coprocessor_error has_error_code=0
idtentry alignment_check do_alignment_check has_error_code=1
idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0
idtentry divide_error do_divide_error has_error_code=0 trapnr=0
idtentry overflow do_overflow has_error_code=0 trapnr=4
idtentry bounds do_bounds has_error_code=0 trapnr=5
idtentry invalid_op do_invalid_op has_error_code=0 trapnr=6
idtentry device_not_available do_device_not_available has_error_code=0 trapnr=7
idtentry double_fault do_double_fault has_error_code=1 paranoid=2 trapnr=8
idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 trapnr=9
idtentry invalid_TSS do_invalid_TSS has_error_code=1 trapnr=10
idtentry segment_not_present do_segment_not_present has_error_code=1 trapnr=11
idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0 trapnr=15
idtentry coprocessor_error do_coprocessor_error has_error_code=0 trapnr=16
idtentry alignment_check do_alignment_check has_error_code=1 trapnr=17
idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0 trapnr=19
/*
......@@ -992,10 +1093,14 @@ bad_gs:
ENTRY(do_softirq_own_stack)
pushq %rbp
mov %rsp, %rbp
HARD_COND_DISABLE_INTERRUPTS
ENTER_IRQ_STACK regs=0 old_rsp=%r11
HARD_COND_ENABLE_INTERRUPTS
call __do_softirq
HARD_COND_DISABLE_INTERRUPTS
LEAVE_IRQ_STACK regs=0
leaveq
HARD_COND_ENABLE_INTERRUPTS
ret
ENDPROC(do_softirq_own_stack)
......@@ -1090,9 +1195,13 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
hyperv_callback_vector hyperv_vector_handler
#endif /* CONFIG_HYPERV */
#ifdef CONFIG_IPIPE
idtentry debug do_debug has_error_code=0 paranoid=1 trapnr=1
#else
idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
idtentry int3 do_int3 has_error_code=0
idtentry stack_segment do_stack_segment has_error_code=1
#endif
idtentry int3 do_int3 has_error_code=0 trapnr=3
idtentry stack_segment do_stack_segment has_error_code=1 trapnr=12
#ifdef CONFIG_XEN
idtentry xennmi do_nmi has_error_code=0
......@@ -1100,15 +1209,15 @@ idtentry xendebug do_debug has_error_code=0
idtentry xenint3 do_int3 has_error_code=0
#endif
idtentry general_protection do_general_protection has_error_code=1
idtentry page_fault do_page_fault has_error_code=1
idtentry general_protection do_general_protection has_error_code=1 trapnr=13
idtentry page_fault do_page_fault has_error_code=1 trapnr=14
#ifdef CONFIG_KVM_GUEST
idtentry async_page_fault do_async_page_fault has_error_code=1
idtentry async_page_fault do_async_page_fault has_error_code=1 trapnr=14
#endif
#ifdef CONFIG_X86_MCE
idtentry machine_check do_mce has_error_code=0 paranoid=1
idtentry machine_check do_mce has_error_code=0 paranoid=1 trapnr=18
#endif
/*
......
......@@ -31,6 +31,7 @@
#ifdef CONFIG_TRACE_IRQFLAGS
THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
THUNK trace_hardirqs_on_virt_thunk,trace_hardirqs_on_virt_caller,1
THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
#endif
......
......@@ -40,6 +40,7 @@
#ifdef CONFIG_TRACE_IRQFLAGS
THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
THUNK trace_hardirqs_on_virt_thunk,trace_hardirqs_on_virt_caller,1
THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
#endif
......
......@@ -14,6 +14,7 @@
*/
#include <linux/timekeeper_internal.h>
#include <linux/ipipe_tickdev.h>
#include <asm/vgtod.h>
#include <asm/vvar.h>
......@@ -75,4 +76,7 @@ void update_vsyscall(struct timekeeper *tk)
}
gtod_write_end(vdata);
if (tk->tkr_mono.clock == &clocksource_tsc)
ipipe_update_hostrt(tk);
}
......@@ -435,7 +435,17 @@ static inline void apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v)) {}
#endif /* CONFIG_X86_LOCAL_APIC */
#ifdef CONFIG_IPIPE
#ifdef CONFIG_SMP
struct irq_data;
void move_xxapic_irq(struct irq_data *data);
#endif
#define ack_APIC_irq() do { } while(0)
static inline void __ack_APIC_irq(void)
#else /* !CONFIG_IPIPE */
#define __ack_APIC_irq() ack_APIC_irq()
static inline void ack_APIC_irq(void)
#endif /* CONFIG_IPIPE */
{
/*
* ack_APIC_irq() actually gets compiled as a single instruction
......
......@@ -94,7 +94,7 @@ extern void aout_dump_debugregs(struct user *dump);
extern void hw_breakpoint_restore(void);
#ifdef CONFIG_X86_64
#if defined(CONFIG_X86_64) && !defined(CONFIG_IPIPE)
DECLARE_PER_CPU(int, debug_stack_usage);
static inline void debug_stack_usage_inc(void)
{
......
......@@ -309,7 +309,7 @@ static inline void force_reload_TR(void)
*/
static inline void refresh_tss_limit(void)
{
DEBUG_LOCKS_WARN_ON(preemptible());
DEBUG_LOCKS_WARN_ON(!hard_irqs_disabled() && preemptible());
if (unlikely(this_cpu_read(__tss_limit_invalid)))
force_reload_TR();
......@@ -391,7 +391,7 @@ void alloc_intr_gate(unsigned int n, const void *addr);
extern unsigned long used_vectors[];
#ifdef CONFIG_X86_64
#if defined(CONFIG_X86_64) && !defined(CONFIG_IPIPE)
DECLARE_PER_CPU(u32, debug_idt_ctr);
static inline bool is_debug_idt_enabled(void)
{
......
......@@ -604,4 +604,24 @@ static inline void xsetbv(u32 index, u64 value)
: : "a" (eax), "d" (edx), "c" (index));
}
DECLARE_PER_CPU(bool, in_kernel_fpu);
static inline void kernel_fpu_disable(void)
{
WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, true);
}
static inline void kernel_fpu_enable(void)
{
WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
this_cpu_write(in_kernel_fpu, false);
}
static inline bool kernel_fpu_disabled(void)
{
return this_cpu_read(in_kernel_fpu);
}
#endif /* _ASM_X86_FPU_INTERNAL_H */
......@@ -26,7 +26,7 @@ extern unsigned int cached_irq_mask;
#define SLAVE_ICW4_DEFAULT 0x01
#define PIC_ICW4_AEOI 2
extern raw_spinlock_t i8259A_lock;
IPIPE_DECLARE_RAW_SPINLOCK(i8259A_lock);
/* the PIC may need a careful delay on some platforms, hence specific calls */
static inline unsigned char inb_pic(unsigned int port)
......
/* -*- linux-c -*-
* arch/x86/include/asm/ipipe.h
*
* Copyright (C) 2007 Philippe Gerum.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef __X86_IPIPE_H
#define __X86_IPIPE_H
#ifdef CONFIG_IPIPE
#define IPIPE_CORE_RELEASE 2112
struct ipipe_domain;
struct ipipe_arch_sysinfo {
};
#define ipipe_processor_id() raw_smp_processor_id()
/* Private interface -- Internal use only */
#define __ipipe_early_core_setup() do { } while(0)
#define __ipipe_enable_irq(irq) irq_to_desc(irq)->chip->enable(irq)
#define __ipipe_disable_irq(irq) irq_to_desc(irq)->chip->disable(irq)
#ifdef CONFIG_SMP
void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
#else
#define __ipipe_hook_critical_ipi(ipd) do { } while(0)
#endif
void __ipipe_enable_pipeline(void);
#define __ipipe_root_tick_p(regs) ((regs)->flags & X86_EFLAGS_IF)
static inline void ipipe_notify_root_preemption(void)
{
__ipipe_notify_vm_preemption();
}
#endif /* CONFIG_IPIPE */
#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE)
#define __ipipe_move_root_irq(__desc) \
do { \
if (!IS_ERR_OR_NULL(__desc)) { \
struct irq_chip *__chip = irq_desc_get_chip(__desc); \
if (__chip->irq_move) \
__chip->irq_move(irq_desc_get_irq_data(__desc)); \
} \
} while (0)
#else /* !(CONFIG_SMP && CONFIG_IPIPE) */
#define __ipipe_move_root_irq(irq) do { } while (0)
#endif /* !(CONFIG_SMP && CONFIG_IPIPE) */
#endif /* !__X86_IPIPE_H */
/* -*- linux-c -*-
* arch/x86/include/asm/ipipe_base.h
*
* Copyright (C) 2007-2012 Philippe Gerum.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef __X86_IPIPE_BASE_H
#define __X86_IPIPE_BASE_H
#include <asm/irq_vectors.h>
#include <asm/bitsperlong.h>
#ifdef CONFIG_X86_32
/* 32 from IDT + iret_error + mayday trap */
#define IPIPE_TRAP_MAYDAY 33 /* Internal recovery trap */
#define IPIPE_NR_FAULTS 34
#else
/* 32 from IDT + mayday trap */
#define IPIPE_TRAP_MAYDAY 32 /* Internal recovery trap */
#define IPIPE_NR_FAULTS 33
#endif
#ifdef CONFIG_X86_LOCAL_APIC
/*
* Special APIC interrupts are mapped above the last defined external
* IRQ number.
*/
#define nr_apic_vectors (NR_VECTORS - FIRST_SYSTEM_VECTOR)
#define IPIPE_FIRST_APIC_IRQ NR_IRQS
#define IPIPE_HRTIMER_IPI ipipe_apic_vector_irq(IPIPE_HRTIMER_VECTOR)
#ifdef CONFIG_SMP
#define IPIPE_RESCHEDULE_IPI ipipe_apic_vector_irq(IPIPE_RESCHEDULE_VECTOR)
#define IPIPE_CRITICAL_IPI ipipe_apic_vector_irq(IPIPE_CRITICAL_VECTOR)
#endif /* CONFIG_SMP */
#define IPIPE_NR_XIRQS (NR_IRQS + nr_apic_vectors)
#define ipipe_apic_irq_vector(irq) ((irq) - IPIPE_FIRST_APIC_IRQ + FIRST_SYSTEM_VECTOR)
#define ipipe_apic_vector_irq(vec) ((vec) - FIRST_SYSTEM_VECTOR + IPIPE_FIRST_APIC_IRQ)
#else
#define IPIPE_NR_XIRQS NR_IRQS
#endif /* !CONFIG_X86_LOCAL_APIC */
#ifndef __ASSEMBLY__
#include <asm/apicdef.h>
extern unsigned int cpu_khz;
static inline const char *ipipe_clock_name(void)
{
return "tsc";
}
#define __ipipe_cpu_freq ({ u64 __freq = 1000ULL * cpu_khz; __freq; })
#define __ipipe_hrclock_freq __ipipe_cpu_freq
#ifdef CONFIG_X86_32
#define ipipe_read_tsc(t) \
__asm__ __volatile__("rdtsc" : "=A"(t))
#define ipipe_tsc2ns(t) \
({ \
unsigned long long delta = (t) * 1000000ULL; \
unsigned long long freq = __ipipe_hrclock_freq; \
do_div(freq, 1000); \
do_div(delta, (unsigned)freq + 1); \
(unsigned long)delta; \
})
#define ipipe_tsc2us(t) \
({ \
unsigned long long delta = (t) * 1000ULL; \
unsigned long long freq = __ipipe_hrclock_freq; \
do_div(freq, 1000); \
do_div(delta, (unsigned)freq + 1); \
(unsigned long)delta; \
})
static inline unsigned long __ipipe_ffnz(unsigned long ul)
{
__asm__("bsrl %1, %0":"=r"(ul) : "r"(ul));
return ul;
}
#else /* X86_64 */
#define ipipe_read_tsc(t) do { \
unsigned int __a,__d; \
asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); \
(t) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \
} while(0)
#define ipipe_tsc2ns(t) (((t) * 1000UL) / (__ipipe_hrclock_freq / 1000000UL))
#define ipipe_tsc2us(t) ((t) / (__ipipe_hrclock_freq / 1000000UL))
static inline unsigned long __ipipe_ffnz(unsigned long ul)
{
__asm__("bsrq %1, %0":"=r"(ul)
: "rm"(ul));
return ul;
}
#endif /* X86_64 */
struct pt_regs;
struct irq_desc;
struct ipipe_vm_notifier;
static inline unsigned __ipipe_get_irq_vector(int irq)
{
#ifdef CONFIG_X86_IO_APIC
unsigned int __ipipe_get_ioapic_irq_vector(int irq);
return __ipipe_get_ioapic_irq_vector(irq);
#elif defined(CONFIG_X86_LOCAL_APIC)
return irq >= IPIPE_FIRST_APIC_IRQ ?
ipipe_apic_irq_vector(irq) : ISA_IRQ_VECTOR(irq);
#else
return ISA_IRQ_VECTOR(irq);
#endif
}
void ipipe_hrtimer_interrupt(void);
void ipipe_reschedule_interrupt(void);
void ipipe_critical_interrupt(void);
int __ipipe_handle_irq(struct pt_regs *regs);
void __ipipe_handle_vm_preemption(struct ipipe_vm_notifier *nfy);
extern int __ipipe_hrtimer_irq;
#endif /* !__ASSEMBLY__ */
#endif /* !__X86_IPIPE_BASE_H */
......@@ -102,6 +102,11 @@
#define POSTED_INTR_NESTED_VECTOR 0xf0
#endif
/* Interrupt pipeline IPIs */
#define IPIPE_HRTIMER_VECTOR 0xee
#define IPIPE_RESCHEDULE_VECTOR 0xed
#define IPIPE_CRITICAL_VECTOR 0xec
/*
* Local APIC timer IRQ vector is on a different priority level,
* to work around the 'lost local interrupt if more than 2 IRQ
......@@ -109,13 +114,13 @@
*/
#define LOCAL_TIMER_VECTOR 0xef
#define NR_VECTORS 256
/*
* I-pipe: Lowest vector number which may be assigned to a special
* APIC IRQ. We must know this at build time.
*/
#define FIRST_SYSTEM_VECTOR IPIPE_CRITICAL_VECTOR
#ifdef CONFIG_X86_LOCAL_APIC
#define FIRST_SYSTEM_VECTOR LOCAL_TIMER_VECTOR
#else
#define FIRST_SYSTEM_VECTOR NR_VECTORS
#endif
#define NR_VECTORS 256
#define FPU_IRQ 13
......
......@@ -6,6 +6,10 @@
#ifndef __ASSEMBLY__
#include <linux/ipipe_trace.h>
#include <linux/compiler.h>
#include <asm-generic/ipipe.h>
/* Provide __cpuidle; we can't safely include <linux/cpu.h> */
#define __cpuidle __attribute__((__section__(".cpuidle.text")))
......@@ -62,14 +66,76 @@ static inline __cpuidle void native_halt(void)
asm volatile("hlt": : :"memory");
}
static inline int native_irqs_disabled(void)
{
unsigned long flags = native_save_fl();
return !(flags & X86_EFLAGS_IF);
}
#endif
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#define HARD_COND_ENABLE_INTERRUPTS
#define HARD_COND_DISABLE_INTERRUPTS
#else
#ifndef __ASSEMBLY__
#include <linux/types.h>
#ifdef CONFIG_IPIPE
void __ipipe_halt_root(int use_mwait);
static inline notrace unsigned long arch_local_save_flags(void)
{
unsigned long flags;
flags = (!ipipe_test_root()) << 9;
barrier();
return flags;
}
static inline notrace void arch_local_irq_restore(unsigned long flags)
{
barrier();
ipipe_restore_root(!(flags & X86_EFLAGS_IF));
}
static inline notrace void arch_local_irq_disable(void)
{
ipipe_stall_root();
barrier();
}
static inline notrace void arch_local_irq_enable(void)
{
barrier();
ipipe_unstall_root();
}
static inline __cpuidle void arch_safe_halt(void)
{
barrier();
__ipipe_halt_root(0);
}
/* Merge virtual+real interrupt mask bits into a single word. */
static inline unsigned long arch_mangle_irq_bits(int virt, unsigned long real)
{
return (real & ~(1L << 31)) | ((unsigned long)(virt != 0) << 31);
}
/* Converse operation of arch_mangle_irq_bits() */
static inline int arch_demangle_irq_bits(unsigned long *x)
{
int virt = (*x & (1L << 31)) != 0;
*x &= ~(1L << 31);
return virt;
}
#else /* !CONFIG_IPIPE */
static inline notrace unsigned long arch_local_save_flags(void)
{
return native_save_fl();
......@@ -99,6 +165,8 @@ static inline __cpuidle void arch_safe_halt(void)
native_safe_halt();
}
#endif /* !CONFIG_IPIPE */
/*
* Used when interrupts are already enabled or to
* shutdown the processor:
......@@ -122,6 +190,14 @@ static inline notrace unsigned long arch_local_irq_save(void)
#define ENABLE_INTERRUPTS(x) sti
#define DISABLE_INTERRUPTS(x) cli
#ifdef CONFIG_IPIPE
#define HARD_COND_ENABLE_INTERRUPTS sti
#define HARD_COND_DISABLE_INTERRUPTS cli
#else /* !CONFIG_IPIPE */
#define HARD_COND_ENABLE_INTERRUPTS
#define HARD_COND_DISABLE_INTERRUPTS
#endif /* !CONFIG_IPIPE */
#ifdef CONFIG_X86_64
#define SWAPGS swapgs
/*
......@@ -170,40 +246,156 @@ static inline int arch_irqs_disabled(void)
return arch_irqs_disabled_flags(flags);
}
#ifdef CONFIG_IPIPE
static inline unsigned long hard_local_irq_save_notrace(void)
{
unsigned long flags;
flags = native_save_fl();
native_irq_disable();
return flags;
}
static inline void hard_local_irq_restore_notrace(unsigned long flags)
{
native_restore_fl(flags);
}
static inline void hard_local_irq_disable_notrace(void)
{
native_irq_disable();
}
static inline void hard_local_irq_enable_notrace(void)
{
native_irq_enable();
}
static inline int hard_irqs_disabled(void)
{
return native_irqs_disabled();
}
#define hard_irqs_disabled_flags(flags) arch_irqs_disabled_flags(flags)
#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
static inline void hard_local_irq_disable(void)
{
if (!native_irqs_disabled()) {
native_irq_disable();
ipipe_trace_begin(0x80000000);
}
}
static inline void hard_local_irq_enable(void)
{
if (native_irqs_disabled()) {
ipipe_trace_end(0x80000000);
native_irq_enable();
}
}
static inline unsigned long hard_local_irq_save(void)
{
unsigned long flags;
flags = native_save_fl();
if (flags & X86_EFLAGS_IF) {
native_irq_disable();
ipipe_trace_begin(0x80000001);
}
return flags;
}
static inline void hard_local_irq_restore(unsigned long flags)
{
if (flags & X86_EFLAGS_IF)
ipipe_trace_end(0x80000001);
native_restore_fl(flags);
}
#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
static inline unsigned long hard_local_irq_save(void)
{
return hard_local_irq_save_notrace();
}
static inline void hard_local_irq_restore(unsigned long flags)
{
hard_local_irq_restore_notrace(flags);
}
static inline void hard_local_irq_enable(void)
{
hard_local_irq_enable_notrace();
}
static inline void hard_local_irq_disable(void)
{
hard_local_irq_disable_notrace();
}
#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
static inline unsigned long hard_local_save_flags(void)
{
return native_save_fl();
}
#endif /* CONFIG_IPIPE */
#endif /* !__ASSEMBLY__ */
#ifdef __ASSEMBLY__
#ifdef CONFIG_TRACE_IRQFLAGS
# define TRACE_IRQS_ON call trace_hardirqs_on_thunk;
#ifdef CONFIG_IPIPE
# define TRACE_IRQS_ON_VIRT call trace_hardirqs_on_virt_thunk;
#else
# define TRACE_IRQS_ON_VIRT TRACE_IRQS_ON
#endif
# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk;
#else
# define TRACE_IRQS_ON
# define TRACE_IRQS_ON_VIRT
# define TRACE_IRQS_OFF
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_X86_64
# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk
# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk
# define LOCKDEP_SYS_EXIT_IRQ \
TRACE_IRQS_ON; \
sti; \
call lockdep_sys_exit_thunk; \
cli; \
TRACE_IRQS_OFF;
# else
# define LOCKDEP_SYS_EXIT \
# define LOCKDEP_SYS_EXIT \
pushl %eax; \
pushl %ecx; \
pushl %edx; \
pushfl; \
sti; \
call lockdep_sys_exit; \
popfl; \
popl %edx; \
popl %ecx; \
popl %eax;
# define LOCKDEP_SYS_EXIT_IRQ
# endif
#else
# define LOCKDEP_SYS_EXIT
# define LOCKDEP_SYS_EXIT_IRQ
#endif
#endif /* __ASSEMBLY__ */
#endif /* __ASSEMBLY__ */
#endif
......@@ -177,7 +177,8 @@ static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
load_mm_ldt(next);
#endif
DEBUG_LOCKS_WARN_ON(preemptible());
DEBUG_LOCKS_WARN_ON(preemptible() &&
(!IS_ENABLED(CONFIG_IPIPE) || !hard_irqs_disabled()));
}
void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);
......@@ -213,6 +214,9 @@ extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk);
#define switch_mm_irqs_off switch_mm_irqs_off
#define ipipe_switch_mm_head(prev, next, tsk) \
switch_mm_irqs_off(prev, next, tsk)
#define activate_mm(prev, next) \
do { \
paravirt_activate_mm((prev), (next)); \
......
......@@ -52,10 +52,15 @@
struct task_struct;
#include <asm/cpufeature.h>
#include <linux/atomic.h>
#include <ipipe/thread_info.h>
struct thread_info {
unsigned long flags; /* low level flags */
u32 status; /* thread synchronous flags */
#ifdef CONFIG_IPIPE
unsigned long ipipe_flags;
struct ipipe_threadinfo ipipe_data;
#endif
};
#define INIT_THREAD_INFO(tsk) \
......@@ -167,6 +172,15 @@ struct thread_info {
#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
/* ti->ipipe_flags */
#define TIP_HEAD 0 /* Runs in head domain */
#define TIP_NOTIFY 1 /* Notify head domain about kernel events */
#define TIP_MAYDAY 2 /* MAYDAY call is pending */
#define _TIP_HEAD (1 << TIP_HEAD)
#define _TIP_NOTIFY (1 << TIP_NOTIFY)
#define _TIP_MAYDAY (1 << TIP_MAYDAY)
#define STACK_WARN (THREAD_SIZE/8)
/*
......
......@@ -15,6 +15,7 @@
*/
typedef unsigned long long cycles_t;
extern struct clocksource clocksource_tsc;
extern unsigned int cpu_khz;
extern unsigned int tsc_khz;
......
......@@ -7,6 +7,7 @@
#include <linux/compiler.h>
#include <linux/kasan-checks.h>
#include <linux/string.h>
#include <linux/ipipe.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
......@@ -70,7 +71,7 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un
})
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define WARN_ON_IN_IRQ() WARN_ON_ONCE(!in_task())
# define WARN_ON_IN_IRQ() WARN_ON_ONCE(ipipe_root_p && !in_task())
#else
# define WARN_ON_IN_IRQ()
#endif
......
......@@ -75,6 +75,7 @@ obj-y += reboot.o
obj-$(CONFIG_X86_MSR) += msr.o
obj-$(CONFIG_X86_CPUID) += cpuid.o
obj-$(CONFIG_PCI) += early-quirks.o
obj-$(CONFIG_IPIPE) += ipipe.o
apm-y := apm_32.o
obj-$(CONFIG_APM) += apm.o
obj-$(CONFIG_SMP) += smp.o
......
......@@ -35,6 +35,7 @@
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/irq.h>
#include <linux/ipipe_tickdev.h>
#include <asm/trace/irq_vectors.h>
#include <asm/irq_remapping.h>
......@@ -269,10 +270,10 @@ void native_apic_icr_write(u32 low, u32 id)
{
unsigned long flags;
local_irq_save(flags);
flags = hard_local_irq_save();
apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
apic_write(APIC_ICR, low);
local_irq_restore(flags);
hard_local_irq_restore(flags);
}
u64 native_apic_icr_read(void)
......@@ -482,16 +483,20 @@ static int lapic_next_deadline(unsigned long delta,
static int lapic_timer_shutdown(struct clock_event_device *evt)
{
unsigned long flags;
unsigned int v;
/* Lapic used as dummy for broadcast ? */
if (evt->features & CLOCK_EVT_FEAT_DUMMY)
return 0;
flags = hard_local_irq_save();
v = apic_read(APIC_LVTT);
v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
apic_write(APIC_LVTT, v);
apic_write(APIC_TMICT, 0);
hard_local_irq_restore(flags);
return 0;
}
......@@ -526,6 +531,17 @@ static void lapic_timer_broadcast(const struct cpumask *mask)
#endif
}
#ifdef CONFIG_IPIPE
static void lapic_itimer_ack(void)
{
__ack_APIC_irq();
}
static DEFINE_PER_CPU(struct ipipe_timer, lapic_itimer) = {
.irq = ipipe_apic_vector_irq(LOCAL_TIMER_VECTOR),
.ack = lapic_itimer_ack,
};
#endif /* CONFIG_IPIPE */
/*
* The local apic timer can be used for any function which is CPU local.
......@@ -656,6 +672,16 @@ static void setup_APIC_timer(void)
memcpy(levt, &lapic_clockevent, sizeof(*levt));
levt->cpumask = cpumask_of(smp_processor_id());
#ifdef CONFIG_IPIPE
if (!(lapic_clockevent.features & CLOCK_EVT_FEAT_DUMMY))
levt->ipipe_timer = this_cpu_ptr(&lapic_itimer);
else {
static atomic_t once = ATOMIC_INIT(-1);
if (atomic_inc_and_test(&once))
printk(KERN_INFO
"I-pipe: cannot use LAPIC as a tick device\n");
}
#endif /* CONFIG_IPIPE */
if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
levt->name = "lapic-deadline";
......@@ -1199,7 +1225,7 @@ void lapic_shutdown(void)
if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
return;
local_irq_save(flags);
flags = hard_local_irq_save();
#ifdef CONFIG_X86_32
if (!enabled_via_apicbase)
......@@ -1209,7 +1235,7 @@ void lapic_shutdown(void)
disable_local_APIC();
local_irq_restore(flags);
hard_local_irq_restore(flags);
}
/**
......@@ -1411,7 +1437,7 @@ void setup_local_APIC(void)
value = apic_read(APIC_ISR + i*0x10);
for (j = 31; j >= 0; j--) {
if (value & (1<<j)) {
ack_APIC_irq();
__ack_APIC_irq();
acked++;
}
}
......@@ -1934,7 +1960,7 @@ __visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs)
*/
v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
if (v & (1 << (vector & 0x1f)))
ack_APIC_irq();
__ack_APIC_irq();
inc_irq_stat(irq_spurious_count);
......@@ -2474,12 +2500,12 @@ static int lapic_suspend(void)
apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
#endif
local_irq_save(flags);
flags = hard_local_irq_save();
disable_local_APIC();
irq_remapping_disable();
local_irq_restore(flags);
hard_local_irq_restore(flags);
return 0;
}
......@@ -2492,7 +2518,7 @@ static void lapic_resume(void)
if (!apic_pm_state.active)
return;
local_irq_save(flags);
flags = hard_local_irq_save();
/*
* IO-APIC and PIC have their own resume routines.
......@@ -2550,7 +2576,7 @@ static void lapic_resume(void)
irq_remapping_reenable(x2apic_mode);
local_irq_restore(flags);
hard_local_irq_restore(flags);
}
/*
......
......@@ -57,9 +57,9 @@ static void _flat_send_IPI_mask(unsigned long mask, int vector)
{
unsigned long flags;
local_irq_save(flags);
flags = hard_local_irq_save();
__default_send_IPI_dest_field(mask, vector, apic->dest_logical);
local_irq_restore(flags);
hard_local_irq_restore(flags);
}
static void flat_send_IPI_mask(const struct cpumask *cpumask, int vector)
......
......@@ -59,7 +59,10 @@ static struct irq_chip ht_irq_chip = {
.irq_ack = irq_chip_ack_parent,
.irq_set_affinity = ht_set_affinity,
.irq_retrigger = irq_chip_retrigger_hierarchy,
.flags = IRQCHIP_SKIP_SET_WAKE,
#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
.irq_move = move_xxapic_irq,
#endif
.flags = IRQCHIP_SKIP_SET_WAKE|IRQCHIP_PIPELINE_SAFE,
};
static int htirq_domain_alloc(struct irq_domain *domain, unsigned int virq,
......
......@@ -77,7 +77,7 @@
#define for_each_irq_pin(entry, head) \
list_for_each_entry(entry, &head, list)
static DEFINE_RAW_SPINLOCK(ioapic_lock);
static IPIPE_DEFINE_RAW_SPINLOCK(ioapic_lock);
static DEFINE_MUTEX(ioapic_mutex);
static unsigned int ioapic_dynirq_base;
static int ioapic_initialized;
......@@ -465,13 +465,19 @@ static void io_apic_sync(struct irq_pin_list *entry)
readl(&io_apic->data);
}
static inline void __mask_ioapic(struct mp_chip_data *data)
{
io_apic_modify_irq(data, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
}
static void mask_ioapic_irq(struct irq_data *irq_data)
{
struct mp_chip_data *data = irq_data->chip_data;
unsigned long flags;
raw_spin_lock_irqsave(&ioapic_lock, flags);
io_apic_modify_irq(data, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
ipipe_lock_irq(irq_data->irq);
__mask_ioapic(data);
raw_spin_unlock_irqrestore(&ioapic_lock, flags);
}
......@@ -487,6 +493,7 @@ static void unmask_ioapic_irq(struct irq_data *irq_data)
raw_spin_lock_irqsave(&ioapic_lock, flags);
__unmask_ioapic(data);
ipipe_unlock_irq(irq_data->irq);
raw_spin_unlock_irqrestore(&ioapic_lock, flags);
}
......@@ -530,14 +537,20 @@ static void __eoi_ioapic_pin(int apic, int pin, int vector)
}
}
static void eoi_ioapic_pin(int vector, struct mp_chip_data *data)
static void _eoi_ioapic_pin(int vector, struct mp_chip_data *data)
{
unsigned long flags;
struct irq_pin_list *entry;
raw_spin_lock_irqsave(&ioapic_lock, flags);
for_each_irq_pin(entry, data->irq_2_pin)
__eoi_ioapic_pin(entry->apic, entry->pin, vector);
}
void eoi_ioapic_pin(int vector, struct mp_chip_data *data)
{
unsigned long flags;
raw_spin_lock_irqsave(&ioapic_lock, flags);
_eoi_ioapic_pin(vector, data);
raw_spin_unlock_irqrestore(&ioapic_lock, flags);
}
......@@ -1202,6 +1215,19 @@ EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
static struct irq_chip ioapic_chip, ioapic_ir_chip;
#ifdef CONFIG_IPIPE
static void startup_legacy_irq(unsigned irq)
{
unsigned long flags;
legacy_pic->mask(irq);
flags = hard_local_irq_save();
__ipipe_unlock_irq(irq);
hard_local_irq_restore(flags);
}
#else /* !CONFIG_IPIPE */
#define startup_legacy_irq(irq) legacy_pic->mask(irq)
#endif /* !CONFIG_IPIPE */
static void __init setup_IO_APIC_irqs(void)
{
unsigned int ioapic, pin;
......@@ -1652,11 +1678,12 @@ static unsigned int startup_ioapic_irq(struct irq_data *data)
raw_spin_lock_irqsave(&ioapic_lock, flags);
if (irq < nr_legacy_irqs()) {
legacy_pic->mask(irq);
startup_legacy_irq(irq);
if (legacy_pic->irq_pending(irq))
was_pending = 1;