Commit cfc8c1d6 authored by Andrey Ryabinin's avatar Andrey Ryabinin Committed by Greg Kroah-Hartman

x86/asm: Use register variable to get stack pointer value

commit 196bd485 upstream.

Currently we use current_stack_pointer() function to get the value
of the stack pointer register. Since commit:

  f5caf621 ("x86/asm: Fix inline asm call constraints for Clang")

... we have a stack register variable declared. It can be used instead of
current_stack_pointer() function which allows to optimize away some
excessive "mov %rsp, %<dst>" instructions:

 -mov    %rsp,%rdx
 -sub    %rdx,%rax
 -cmp    $0x3fff,%rax
 -ja     ffffffff810722fd <ist_begin_non_atomic+0x2d>

 +sub    %rsp,%rax
 +cmp    $0x3fff,%rax
 +ja     ffffffff810722fa <ist_begin_non_atomic+0x2a>

Remove current_stack_pointer(), rename __asm_call_sp to current_stack_pointer
and use it instead of the removed function.
Signed-off-by: default avatarAndrey Ryabinin <>
Reviewed-by: default avatarJosh Poimboeuf <>
Cc: Andy Lutomirski <>
Cc: Linus Torvalds <>
Cc: Peter Zijlstra <>
Cc: Thomas Gleixner <>
Link: default avatarIngo Molnar <>
[dwmw2: We want ASM_CALL_CONSTRAINT for retpoline]
Signed-off-by: default avatarDavid Woodhouse <>
Signed-off-by: default avatarRazvan Ghitulete <>
Signed-off-by: default avatarGreg Kroah-Hartman <>
parent 416f6650
......@@ -105,4 +105,15 @@
/* For C file, we already have NOKPROBE_SYMBOL macro */
#ifndef __ASSEMBLY__
* This output constraint should be used for any inline asm which has a "call"
* instruction. Otherwise the asm may be inserted before the frame pointer
* gets set up by the containing function. If you forget to do this, objtool
* may print a "call without frame pointer save/setup" warning.
register unsigned long current_stack_pointer asm(_ASM_SP);
#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer)
#endif /* _ASM_X86_ASM_H */
......@@ -166,17 +166,6 @@ static inline struct thread_info *current_thread_info(void)
return (struct thread_info *)(current_top_of_stack() - THREAD_SIZE);
static inline unsigned long current_stack_pointer(void)
unsigned long sp;
#ifdef CONFIG_X86_64
asm("mov %%rsp,%0" : "=g" (sp));
asm("mov %%esp,%0" : "=g" (sp));
return sp;
#else /* !__ASSEMBLY__ */
#ifdef CONFIG_X86_64
......@@ -65,7 +65,7 @@ static void call_on_stack(void *func, void *stack)
static inline void *current_stack(void)
return (void *)(current_stack_pointer() & ~(THREAD_SIZE - 1));
return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
......@@ -89,7 +89,7 @@ static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
/* Save the next esp at the bottom of the stack */
prev_esp = (u32 *)irqstk;
*prev_esp = current_stack_pointer();
*prev_esp = current_stack_pointer;
if (unlikely(overflow))
call_on_stack(print_stack_overflow, isp);
......@@ -142,7 +142,7 @@ void do_softirq_own_stack(void)
/* Push the previous esp onto the stack */
prev_esp = (u32 *)irqstk;
*prev_esp = current_stack_pointer();
*prev_esp = current_stack_pointer;
call_on_stack(__do_softirq, isp);
......@@ -166,7 +166,7 @@ void ist_begin_non_atomic(struct pt_regs *regs)
* from double_fault.
BUG_ON((unsigned long)(current_top_of_stack() -
current_stack_pointer()) >= THREAD_SIZE);
current_stack_pointer) >= THREAD_SIZE);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment