Commit d9f057db authored by Philippe Gerum's avatar Philippe Gerum

genirq: add generic I-pipe core

This commit provides the arch-independent bits for implementing the
interrupt pipeline core, a lightweight layer introducing a separate,
high-priority execution stage for handling all IRQs in pseudo-NMI
mode, which cannot be delayed by the regular kernel code. See
Documentation/ipipe.rst for details about interrupt pipelining.

Architectures which support interrupt pipelining should select
HAVE_IPIPE_SUPPORT, along with implementing the required arch-specific
code. In such a case, CONFIG_IPIPE becomes available to the user via
the Kconfig interface for enabling the feature.
parent e7405910
This diff is collapsed.
/* -*- linux-c -*-
* include/asm-generic/ipipe.h
*
* Copyright (C) 2002-2017 Philippe Gerum.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*/
#ifndef __ASM_GENERIC_IPIPE_H
#define __ASM_GENERIC_IPIPE_H
#ifdef CONFIG_IPIPE
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || defined(CONFIG_PROVE_LOCKING) || \
defined(CONFIG_PREEMPT_VOLUNTARY) || defined(CONFIG_IPIPE_DEBUG_CONTEXT)
void __ipipe_uaccess_might_fault(void);
#else
#define __ipipe_uaccess_might_fault() might_fault()
#endif
#define hard_cond_local_irq_enable() hard_local_irq_enable()
#define hard_cond_local_irq_disable() hard_local_irq_disable()
#define hard_cond_local_irq_save() hard_local_irq_save()
#define hard_cond_local_irq_restore(flags) hard_local_irq_restore(flags)
#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
void ipipe_root_only(void);
#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
static inline void ipipe_root_only(void) { }
#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
void ipipe_stall_root(void);
void ipipe_unstall_root(void);
unsigned long ipipe_test_and_stall_root(void);
unsigned long ipipe_test_root(void);
void ipipe_restore_root(unsigned long x);
#else /* !CONFIG_IPIPE */
#define hard_local_irq_save_notrace() \
({ \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags; \
})
#define hard_local_irq_restore_notrace(__flags) \
raw_local_irq_restore(__flags)
#define hard_local_irq_enable_notrace() \
raw_local_irq_enable()
#define hard_local_irq_disable_notrace() \
raw_local_irq_disable()
#define hard_local_irq_save() \
({ \
unsigned long __flags; \
local_irq_save(__flags); \
__flags; \
})
#define hard_local_irq_restore(__flags) local_irq_restore(__flags)
#define hard_local_irq_enable() local_irq_enable()
#define hard_local_irq_disable() local_irq_disable()
#define hard_irqs_disabled() irqs_disabled()
#define hard_cond_local_irq_enable() do { } while(0)
#define hard_cond_local_irq_disable() do { } while(0)
#define hard_cond_local_irq_save() 0
#define hard_cond_local_irq_restore(__flags) do { (void)(__flags); } while(0)
#define __ipipe_uaccess_might_fault() might_fault()
static inline void ipipe_root_only(void) { }
#endif /* !CONFIG_IPIPE */
#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE)
#define hard_smp_local_irq_save() hard_local_irq_save()
#define hard_smp_local_irq_restore(__flags) hard_local_irq_restore(__flags)
#else /* !CONFIG_SMP */
#define hard_smp_local_irq_save() 0
#define hard_smp_local_irq_restore(__flags) do { (void)(__flags); } while(0)
#endif /* CONFIG_SMP */
#endif
......@@ -5,6 +5,7 @@
#include <linux/compiler.h>
#include <linux/threads.h>
#include <linux/percpu-defs.h>
#include <asm-generic/ipipe.h>
#ifdef CONFIG_SMP
......@@ -44,11 +45,29 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
#endif
#ifdef CONFIG_IPIPE
#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP)
unsigned long __ipipe_cpu_get_offset(void);
#define __ipipe_cpu_offset __ipipe_cpu_get_offset()
#else
#define __ipipe_cpu_offset __my_cpu_offset
#endif
#ifndef __ipipe_raw_cpu_ptr
#define __ipipe_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __ipipe_cpu_offset)
#endif
#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var)))
#endif /* CONFIG_IPIPE */
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
extern void setup_per_cpu_areas(void);
#endif
#endif /* SMP */
#else /* !SMP */
#define __ipipe_raw_cpu_ptr(ptr) VERIFY_PERCPU_PTR(ptr)
#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var)))
#endif /* !SMP */
#ifndef PER_CPU_BASE_SECTION
#ifdef CONFIG_SMP
......@@ -148,9 +167,9 @@ do { \
#define this_cpu_generic_to_op(pcp, val, op) \
do { \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags = hard_local_irq_save(); \
raw_cpu_generic_to_op(pcp, val, op); \
raw_local_irq_restore(__flags); \
hard_local_irq_restore(__flags); \
} while (0)
......@@ -158,9 +177,9 @@ do { \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags = hard_local_irq_save(); \
__ret = raw_cpu_generic_add_return(pcp, val); \
raw_local_irq_restore(__flags); \
hard_local_irq_restore(__flags); \
__ret; \
})
......@@ -168,9 +187,9 @@ do { \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags = hard_local_irq_save(); \
__ret = raw_cpu_generic_xchg(pcp, nval); \
raw_local_irq_restore(__flags); \
hard_local_irq_restore(__flags); \
__ret; \
})
......@@ -178,9 +197,9 @@ do { \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags = hard_local_irq_save(); \
__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
raw_local_irq_restore(__flags); \
hard_local_irq_restore(__flags); \
__ret; \
})
......@@ -188,10 +207,10 @@ do { \
({ \
int __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__flags = hard_local_irq_save(); \
__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
oval1, oval2, nval1, nval2); \
raw_local_irq_restore(__flags); \
hard_local_irq_restore(__flags); \
__ret; \
})
......
......@@ -6,6 +6,7 @@
#include <linux/lockdep.h>
#include <linux/ftrace_irq.h>
#include <linux/vtime.h>
#include <linux/ipipe.h>
#include <asm/hardirq.h>
......@@ -62,6 +63,7 @@ extern void irq_exit(void);
#define nmi_enter() \
do { \
__ipipe_nmi_enter(); \
printk_nmi_enter(); \
lockdep_off(); \
ftrace_nmi_enter(); \
......@@ -80,6 +82,7 @@ extern void irq_exit(void);
ftrace_nmi_exit(); \
lockdep_on(); \
printk_nmi_exit(); \
__ipipe_nmi_exit(); \
} while (0)
#endif /* LINUX_HARDIRQ_H */
/* -*- linux-c -*-
* include/linux/ipipe.h
*
* Copyright (C) 2002-2014 Philippe Gerum.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef __LINUX_IPIPE_H
#define __LINUX_IPIPE_H
#include <linux/spinlock.h>
#include <linux/cache.h>
#include <linux/percpu.h>
#include <linux/irq.h>
#include <linux/thread_info.h>
#include <linux/ipipe_base.h>
#include <linux/ipipe_debug.h>
#include <asm/ptrace.h>
#ifdef CONFIG_HAVE_IPIPE_SUPPORT
#include <asm/ipipe.h>
#endif
#ifdef CONFIG_IPIPE
#include <linux/ipipe_domain.h>
struct ipipe_sysinfo {
int sys_nr_cpus; /* Number of CPUs on board */
int sys_hrtimer_irq; /* hrtimer device IRQ */
u64 sys_hrtimer_freq; /* hrtimer device frequency */
u64 sys_hrclock_freq; /* hrclock device frequency */
u64 sys_cpu_freq; /* CPU frequency (Hz) */
struct ipipe_arch_sysinfo arch;
};
struct ipipe_work_header {
size_t size;
void (*handler)(struct ipipe_work_header *work);
};
extern unsigned int __ipipe_printk_virq;
void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq);
/*
* Obsolete - no arch implements PIC muting anymore. Null helpers are
* kept for building legacy co-kernel releases.
*/
static inline void ipipe_mute_pic(void) { }
static inline void ipipe_unmute_pic(void) { }
static inline void __ipipe_nmi_enter(void)
{
__this_cpu_write(ipipe_percpu.nmi_state, __ipipe_root_status);
__set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
ipipe_save_context_nmi();
}
static inline void __ipipe_nmi_exit(void)
{
ipipe_restore_context_nmi();
if (!test_bit(IPIPE_STALL_FLAG, raw_cpu_ptr(&ipipe_percpu.nmi_state)))
__clear_bit(IPIPE_STALL_FLAG, &__ipipe_root_status);
}
static inline void __ipipe_sync_pipeline(struct ipipe_domain *top)
{
if (__ipipe_current_domain != top) {
__ipipe_do_sync_pipeline(top);
return;
}
if (!test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(top)->status))
__ipipe_sync_stage();
}
void ipipe_register_head(struct ipipe_domain *ipd,
const char *name);
void ipipe_unregister_head(struct ipipe_domain *ipd);
int ipipe_request_irq(struct ipipe_domain *ipd,
unsigned int irq,
ipipe_irq_handler_t handler,
void *cookie,
ipipe_irq_ackfn_t ackfn);
void ipipe_free_irq(struct ipipe_domain *ipd,
unsigned int irq);
void ipipe_raise_irq(unsigned int irq);
unsigned int ipipe_alloc_virq(void);
void ipipe_free_virq(unsigned int virq);
static inline void ipipe_post_irq_head(unsigned int irq)
{
__ipipe_set_irq_pending(ipipe_head_domain, irq);
}
static inline void ipipe_post_irq_root(unsigned int irq)
{
__ipipe_set_irq_pending(&ipipe_root, irq);
}
static inline void ipipe_stall_head(void)
{
hard_local_irq_disable();
__set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
}
static inline unsigned long ipipe_test_and_stall_head(void)
{
hard_local_irq_disable();
return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
}
static inline unsigned long ipipe_test_head(void)
{
unsigned long flags, ret;
flags = hard_smp_local_irq_save();
ret = test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status);
hard_smp_local_irq_restore(flags);
return ret;
}
void ipipe_unstall_head(void);
void __ipipe_restore_head(unsigned long x);
static inline void ipipe_restore_head(unsigned long x)
{
ipipe_check_irqoff();
if ((x ^ test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)) & 1)
__ipipe_restore_head(x);
}
void __ipipe_post_work_root(struct ipipe_work_header *work);
#define ipipe_post_work_root(p, header) \
do { \
void header_not_at_start(void); \
if (offsetof(typeof(*(p)), header)) { \
header_not_at_start(); \
} \
__ipipe_post_work_root(&(p)->header); \
} while (0)
int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
unsigned long ipipe_critical_enter(void (*syncfn)(void));
void ipipe_critical_exit(unsigned long flags);
void ipipe_prepare_panic(void);
#ifdef CONFIG_SMP
#ifndef ipipe_smp_p
#define ipipe_smp_p (1)
#endif
void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask);
void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask);
#else /* !CONFIG_SMP */
#define ipipe_smp_p (0)
static inline
void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask) { }
static inline void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask) { }
static inline void ipipe_disable_smp(void) { }
#endif /* CONFIG_SMP */
static inline void ipipe_restore_root_nosync(unsigned long x)
{
unsigned long flags;
flags = hard_smp_local_irq_save();
__ipipe_restore_root_nosync(x);
hard_smp_local_irq_restore(flags);
}
/* Must be called hw IRQs off. */
static inline void ipipe_lock_irq(unsigned int irq)
{
struct ipipe_domain *ipd = __ipipe_current_domain;
if (ipd == ipipe_root_domain)
__ipipe_lock_irq(irq);
}
/* Must be called hw IRQs off. */
static inline void ipipe_unlock_irq(unsigned int irq)
{
struct ipipe_domain *ipd = __ipipe_current_domain;
if (ipd == ipipe_root_domain)
__ipipe_unlock_irq(irq);
}
void ipipe_enable_irq(unsigned int irq);
static inline void ipipe_disable_irq(unsigned int irq)
{
struct irq_desc *desc;
struct irq_chip *chip;
desc = irq_to_desc(irq);
if (desc == NULL)
return;
chip = irq_desc_get_chip(desc);
if (WARN_ON_ONCE(chip->irq_disable == NULL && chip->irq_mask == NULL))
return;
if (chip->irq_disable)
chip->irq_disable(&desc->irq_data);
else
chip->irq_mask(&desc->irq_data);
}
static inline void ipipe_end_irq(unsigned int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
if (desc)
desc->ipipe_end(desc);
}
static inline int ipipe_chained_irq_p(struct irq_desc *desc)
{
void __ipipe_chained_irq(struct irq_desc *desc);
return desc->handle_irq == __ipipe_chained_irq;
}
static inline void ipipe_handle_demuxed_irq(unsigned int cascade_irq)
{
__ipipe_dispatch_irq(cascade_irq, IPIPE_IRQF_NOSYNC);
}
static inline void __ipipe_init_threadflags(struct thread_info *ti)
{
ti->ipipe_flags = 0;
}
static inline
void ipipe_set_ti_thread_flag(struct thread_info *ti, int flag)
{
set_bit(flag, &ti->ipipe_flags);
}
static inline
void ipipe_clear_ti_thread_flag(struct thread_info *ti, int flag)
{
clear_bit(flag, &ti->ipipe_flags);
}
static inline
void ipipe_test_and_clear_ti_thread_flag(struct thread_info *ti, int flag)
{
test_and_clear_bit(flag, &ti->ipipe_flags);
}
static inline
int ipipe_test_ti_thread_flag(struct thread_info *ti, int flag)
{
return test_bit(flag, &ti->ipipe_flags);
}
#define ipipe_set_thread_flag(flag) \
ipipe_set_ti_thread_flag(current_thread_info(), flag)
#define ipipe_clear_thread_flag(flag) \
ipipe_clear_ti_thread_flag(current_thread_info(), flag)
#define ipipe_test_and_clear_thread_flag(flag) \
ipipe_test_and_clear_ti_thread_flag(current_thread_info(), flag)
#define ipipe_test_thread_flag(flag) \
ipipe_test_ti_thread_flag(current_thread_info(), flag)
#else /* !CONFIG_IPIPE */
#define __ipipe_root_p 1
#define ipipe_root_p 1
static inline void __ipipe_init_threadflags(struct thread_info *ti) { }
static inline void __ipipe_nmi_enter(void) { }
static inline void __ipipe_nmi_exit(void) { }
#define ipipe_processor_id() smp_processor_id()
static inline void ipipe_lock_irq(unsigned int irq) { }
static inline void ipipe_unlock_irq(unsigned int irq) { }
#endif /* !CONFIG_IPIPE */
#endif /* !__LINUX_IPIPE_H */
/* -*- linux-c -*-
* include/linux/ipipe_base.h
*
* Copyright (C) 2002-2014 Philippe Gerum.
* 2007 Jan Kiszka.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef __LINUX_IPIPE_BASE_H
#define __LINUX_IPIPE_BASE_H
struct irq_desc;
#ifdef CONFIG_IPIPE
#define IPIPE_CORE_APIREV CONFIG_IPIPE_CORE_APIREV
#include <linux/ipipe_domain.h>
#include <linux/compiler.h>
#include <linux/linkage.h>
#include <asm/ipipe_base.h>
struct pt_regs;
struct ipipe_domain;
static inline int ipipe_virtual_irq_p(unsigned int irq)
{
return irq >= IPIPE_VIRQ_BASE && irq < IPIPE_NR_IRQS;
}
void __ipipe_init_early(void);
void __ipipe_init(void);
#ifdef CONFIG_PROC_FS
void __ipipe_init_proc(void);
#ifdef CONFIG_IPIPE_TRACE
void __ipipe_init_tracer(void);
#else /* !CONFIG_IPIPE_TRACE */
static inline void __ipipe_init_tracer(void) { }
#endif /* CONFIG_IPIPE_TRACE */
#else /* !CONFIG_PROC_FS */
static inline void __ipipe_init_proc(void) { }
#endif /* CONFIG_PROC_FS */
void __ipipe_restore_root_nosync(unsigned long x);
#define IPIPE_IRQF_NOACK 0x1
#define IPIPE_IRQF_NOSYNC 0x2
void __ipipe_dispatch_irq(unsigned int irq, int flags);
void __ipipe_do_sync_stage(void);
void __ipipe_do_sync_pipeline(struct ipipe_domain *top);
void __ipipe_lock_irq(unsigned int irq);
void __ipipe_unlock_irq(unsigned int irq);
void __ipipe_do_critical_sync(unsigned int irq, void *cookie);
void __ipipe_ack_edge_irq(struct irq_desc *desc);
void __ipipe_nop_irq(struct irq_desc *desc);
static inline void __ipipe_idle(void)
{
ipipe_unstall_root();
}
#ifndef __ipipe_sync_check
#define __ipipe_sync_check 1
#endif
static inline void __ipipe_sync_stage(void)
{
if (likely(__ipipe_sync_check))
__ipipe_do_sync_stage();
}
#ifndef __ipipe_run_irqtail
#define __ipipe_run_irqtail(irq) do { } while(0)
#endif
int __ipipe_log_printk(const char *fmt, va_list args);
void __ipipe_flush_printk(unsigned int irq, void *cookie);
#define __ipipe_serial_debug(__fmt, __args...) raw_printk(__fmt, ##__args)
#else /* !CONFIG_IPIPE */
struct task_struct;
struct mm_struct;
static inline void __ipipe_init_early(void) { }
static inline void __ipipe_init(void) { }
static inline void __ipipe_init_proc(void) { }
static inline void __ipipe_idle(void) { }
#define __ipipe_root_tick_p(regs) 1
#define ipipe_handle_domain_irq(__domain, __hwirq, __regs) \
handle_domain_irq(__domain, __hwirq, __regs)
#define ipipe_handle_demuxed_irq(irq) generic_handle_irq(irq)
#define __ipipe_serial_debug(__fmt, __args...) do { } while (0)
#endif /* !CONFIG_IPIPE */
#ifdef CONFIG_IPIPE_WANT_PTE_PINNING
void __ipipe_pin_mapping_globally(unsigned long start,
unsigned long end);
#else
static inline void __ipipe_pin_mapping_globally(unsigned long start,
unsigned long end)
{ }
#endif
#endif /* !__LINUX_IPIPE_BASE_H */
/* -*- linux-c -*-
* include/linux/ipipe_debug.h
*
* Copyright (C) 2012 Philippe Gerum <rpm@xenomai.org>.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
* USA; either version 2 of the License, or (at your option) any later
* version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#ifndef __LINUX_IPIPE_DEBUG_H
#define __LINUX_IPIPE_DEBUG_H
#include <linux/ipipe_domain.h>
#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
#include <asm/bug.h>
static inline int ipipe_disable_context_check(void)
{
return xchg(raw_cpu_ptr(&ipipe_percpu.context_check), 0);
}
static inline void ipipe_restore_context_check(int old_state)
{
__this_cpu_write(ipipe_percpu.context_check, old_state);
}
static inline void ipipe_context_check_off(void)
{
int cpu;
for_each_online_cpu(cpu)
per_cpu(ipipe_percpu, cpu).context_check = 0;
}
static inline void ipipe_save_context_nmi(void)
{
int state = ipipe_disable_context_check();
__this_cpu_write(ipipe_percpu.context_check_saved, state);
}
static inline void ipipe_restore_context_nmi(void)
{
ipipe_restore_context_check(__this_cpu_read(ipipe_percpu.context_check_saved));
}
#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
static inline int ipipe_disable_context_check(void)
{
return 0;
}
static inline void ipipe_restore_context_check(int old_state) { }
static inline void ipipe_context_check_off(void) { }
static inline void ipipe_save_context_nmi(void) { }
static inline void ipipe_restore_context_nmi(void) { }
#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
#ifdef CONFIG_IPIPE_DEBUG
#define ipipe_check_irqoff() \
do { \
if (WARN_ON_ONCE(!hard_irqs_disabled())) \
hard_local_irq_disable(); \
} while (0)
#else /* !CONFIG_IPIPE_DEBUG */