Commit 9f70e805 authored by Philippe Gerum's avatar Philippe Gerum

arm64: fpsimd: ipipe: enable sharing with head domain

Serialize accesses to the fpsimd so as to allow co-kernel activities
(e.g. tasks) running in the head domain to share the unit with the
host kernel.

The companion kernel should call fpsimd_restore_current_state() to
restore the FPU context from its own scheduling tail code.
Signed-off-by: Philippe Gerum's avatarPhilippe Gerum <rpm@xenomai.org>
parent 83efbeba
......@@ -38,6 +38,7 @@
#include <linux/slab.h>
#include <linux/stddef.h>
#include <linux/sysctl.h>
#include <linux/ipipe.h>
#include <asm/esr.h>
#include <asm/fpsimd.h>
......@@ -126,6 +127,8 @@ static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
/* Default VL for tasks that don't set it explicitly: */
static int sve_default_vl = -1;
static void __fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st);
#ifdef CONFIG_ARM64_SVE
/* Maximum supported vector length across all CPUs (initially poisoned) */
......@@ -142,6 +145,34 @@ extern void __percpu *efi_sve_state;
#endif /* ! CONFIG_ARM64_SVE */
#ifdef CONFIG_IPIPE
#define fpsimd_enter_atomic(__flags) \
do { \
(__flags) = hard_preempt_disable(); \
} while (0)
#define fpsimd_exit_atomic(__flags) \
do { \
hard_preempt_enable(__flags); \
} while (0)
#else /* !CONFIG_IPIPE */
#define fpsimd_enter_atomic(__flags) \
do { \
local_bh_disable(); \
(void)__flags; \
} while (0)
#define fpsimd_exit_atomic(__flags) \
do { \
local_bh_enable(); \
(void)__flags; \
} while (0)
#endif /* !CONFIG_IPIPE */
/*
* Call __sve_free() directly only if you know task can't be scheduled
* or preempted.
......@@ -217,7 +248,7 @@ static void sve_free(struct task_struct *task)
*/
static void task_fpsimd_load(void)
{
WARN_ON(!in_softirq() && !irqs_disabled());
WARN_ON(!hard_irqs_disabled() && !in_softirq() && !irqs_disabled());
if (system_supports_sve() && test_thread_flag(TIF_SVE))
sve_load_state(sve_pffr(&current->thread),
......@@ -233,12 +264,12 @@ static void task_fpsimd_load(void)
*
* Softirqs (and preemption) must be disabled.
*/
void fpsimd_save(void)
static void __fpsimd_save(void)
{
struct user_fpsimd_state *st = __this_cpu_read(fpsimd_last_state.st);
/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
WARN_ON(!in_softirq() && !irqs_disabled());
WARN_ON(!hard_irqs_disabled() && !in_softirq() && !irqs_disabled());
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
......@@ -258,6 +289,15 @@ void fpsimd_save(void)
}
}
void fpsimd_save(void)
{
unsigned long flags;
flags = hard_cond_local_irq_save();
__fpsimd_save();
hard_cond_local_irq_restore(flags);
}
/*
* Helpers to translate bit indices in sve_vq_map to VQ values (and
* vice versa). This allows find_next_bit() to be used to find the
......@@ -366,7 +406,7 @@ static int __init sve_sysctl_init(void) { return 0; }
* task->thread.uw.fpsimd_state must be up to date before calling this
* function.
*/
static void fpsimd_to_sve(struct task_struct *task)
static void __fpsimd_to_sve(struct task_struct *task)
{
unsigned int vq;
void *sst = task->thread.sve_state;
......@@ -382,6 +422,15 @@ static void fpsimd_to_sve(struct task_struct *task)
sizeof(fst->vregs[i]));
}
static void fpsimd_to_sve(struct task_struct *task)
{
unsigned long flags;
flags = hard_cond_local_irq_save();
__fpsimd_to_sve(task);
hard_cond_local_irq_restore(flags);
}
/*
* Transfer the SVE state in task->thread.sve_state to
* task->thread.uw.fpsimd_state.
......@@ -398,14 +447,19 @@ static void sve_to_fpsimd(struct task_struct *task)
void const *sst = task->thread.sve_state;
struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
unsigned int i;
unsigned long flags;
if (!system_supports_sve())
return;
flags = hard_cond_local_irq_save();
vq = sve_vq_from_vl(task->thread.sve_vl);
for (i = 0; i < 32; ++i)
memcpy(&fst->vregs[i], ZREG(sst, vq, i),
sizeof(fst->vregs[i]));
hard_cond_local_irq_restore(flags);
}
#ifdef CONFIG_ARM64_SVE
......@@ -510,6 +564,8 @@ void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
int sve_set_vector_length(struct task_struct *task,
unsigned long vl, unsigned long flags)
{
unsigned long irqflags = 0;
if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
PR_SVE_SET_VL_ONEXEC))
return -EINVAL;
......@@ -547,9 +603,9 @@ int sve_set_vector_length(struct task_struct *task,
* non-SVE thread.
*/
if (task == current) {
local_bh_disable();
fpsimd_enter_atomic(irqflags);
fpsimd_save();
__fpsimd_save();
set_thread_flag(TIF_FOREIGN_FPSTATE);
}
......@@ -558,7 +614,7 @@ int sve_set_vector_length(struct task_struct *task,
sve_to_fpsimd(task);
if (task == current)
local_bh_enable();
fpsimd_exit_atomic(irqflags);
/*
* Force reallocation of task SVE state to the correct size
......@@ -805,6 +861,8 @@ void fpsimd_release_task(struct task_struct *dead_task)
*/
asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
{
unsigned long flags;
/* Even if we chose not to use SVE, the hardware could still trap: */
if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
......@@ -813,10 +871,10 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
sve_alloc(current);
local_bh_disable();
fpsimd_enter_atomic(flags);
fpsimd_save();
fpsimd_to_sve(current);
__fpsimd_save();
__fpsimd_to_sve(current);
/* Force ret_to_user to reload the registers: */
fpsimd_flush_task_state(current);
......@@ -825,7 +883,7 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
if (test_and_set_thread_flag(TIF_SVE))
WARN_ON(1); /* SVE access shouldn't have trapped */
local_bh_enable();
fpsimd_exit_atomic(flags);
}
/*
......@@ -875,12 +933,15 @@ asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
void fpsimd_thread_switch(struct task_struct *next)
{
bool wrong_task, wrong_cpu;
unsigned long flags;
if (!system_supports_fpsimd())
return;
flags = hard_cond_local_irq_save();
/* Save unsaved fpsimd state, if any: */
fpsimd_save();
__fpsimd_save();
/*
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
......@@ -893,16 +954,19 @@ void fpsimd_thread_switch(struct task_struct *next)
update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
wrong_task || wrong_cpu);
hard_cond_local_irq_restore(flags);
}
void fpsimd_flush_thread(void)
{
int vl, supported_vl;
unsigned long flags;
if (!system_supports_fpsimd())
return;
local_bh_disable();
fpsimd_enter_atomic(flags);
memset(&current->thread.uw.fpsimd_state, 0,
sizeof(current->thread.uw.fpsimd_state));
......@@ -945,7 +1009,7 @@ void fpsimd_flush_thread(void)
set_thread_flag(TIF_FOREIGN_FPSTATE);
local_bh_enable();
fpsimd_exit_atomic(flags);
}
/*
......@@ -954,12 +1018,14 @@ void fpsimd_flush_thread(void)
*/
void fpsimd_preserve_current_state(void)
{
unsigned long flags;
if (!system_supports_fpsimd())
return;
local_bh_disable();
fpsimd_save();
local_bh_enable();
fpsimd_enter_atomic(flags);
__fpsimd_save();
fpsimd_exit_atomic(flags);
}
/*
......@@ -997,14 +1063,23 @@ void fpsimd_bind_task_to_cpu(void)
}
}
void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
static void __fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
{
struct fpsimd_last_state_struct *last =
this_cpu_ptr(&fpsimd_last_state);
last->st = st;
}
void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
{
unsigned long flags;
WARN_ON(!in_softirq() && !irqs_disabled());
last->st = st;
flags = hard_cond_local_irq_save();
__fpsimd_bind_state_to_cpu(st);
hard_cond_local_irq_restore(flags);
}
/*
......@@ -1014,17 +1089,19 @@ void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st)
*/
void fpsimd_restore_current_state(void)
{
unsigned long flags;
if (!system_supports_fpsimd())
return;
local_bh_disable();
fpsimd_enter_atomic(flags);
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
task_fpsimd_load();
fpsimd_bind_task_to_cpu();
}
local_bh_enable();
fpsimd_exit_atomic(flags);
}
/*
......@@ -1034,21 +1111,23 @@ void fpsimd_restore_current_state(void)
*/
void fpsimd_update_current_state(struct user_fpsimd_state const *state)
{
unsigned long flags;
if (!system_supports_fpsimd())
return;
local_bh_disable();
fpsimd_enter_atomic(flags);
current->thread.uw.fpsimd_state = *state;
if (system_supports_sve() && test_thread_flag(TIF_SVE))
fpsimd_to_sve(current);
__fpsimd_to_sve(current);
task_fpsimd_load();
fpsimd_bind_task_to_cpu();
clear_thread_flag(TIF_FOREIGN_FPSTATE);
local_bh_enable();
fpsimd_exit_atomic(flags);
}
/*
......@@ -1089,22 +1168,26 @@ EXPORT_PER_CPU_SYMBOL(kernel_neon_busy);
*/
void kernel_neon_begin(void)
{
unsigned long flags;
if (WARN_ON(!system_supports_fpsimd()))
return;
BUG_ON(!may_use_simd());
flags = hard_preempt_disable();
local_bh_disable();
__this_cpu_write(kernel_neon_busy, true);
/* Save unsaved fpsimd state, if any: */
fpsimd_save();
__fpsimd_save();
/* Invalidate any task state remaining in the fpsimd regs: */
fpsimd_flush_cpu_state();
preempt_disable();
hard_cond_local_irq_restore(flags);
local_bh_enable();
}
......@@ -1130,6 +1213,7 @@ void kernel_neon_end(void)
WARN_ON(!busy); /* No matching kernel_neon_begin()? */
preempt_enable();
hard_cond_local_irq_enable();
}
EXPORT_SYMBOL(kernel_neon_end);
......@@ -1219,10 +1303,14 @@ void __efi_fpsimd_end(void)
static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
unsigned long cmd, void *v)
{
unsigned long flags;
switch (cmd) {
case CPU_PM_ENTER:
fpsimd_save();
flags = hard_cond_local_irq_save();
__fpsimd_save();
fpsimd_flush_cpu_state();
hard_cond_local_irq_restore(flags);
break;
case CPU_PM_EXIT:
break;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment