From: Arjan van de Ven Below is a patch to enable 4Kb stacks for x86. The goal of this is to 1) Reduce footprint per thread so that systems can run many more threads (for the java people) 2) Reduce the pressure on the VM for order > 0 allocations. We see real life workloads (granted with 2.4 but the fundamental fragmentation issue isn't solved in 2.6 and isn't solvable in theory) where this can be a problem. In addition order > 0 allocations can make the VM "stutter" and give more latency due to having to do much much more work trying to defragment The first 2 bits of the patch actually affect compileroptions in a generic way: I propose to disable the -funit-at-a-time feature from gcc. With this enabled (and it's default with -O2), gcc will very agressively inline functions, which is nice and all for userspace, but for the kernel this makes us suffer a gcc deficiency more: gcc is extremely bad at sharing stackslots, for example a situation like this: if (some_condition) function_A(); else function_B(); with -funit-at-a-time, both function_A() and _B() might get inlined, however the stack usage of both functions of the parent function grows the stack usage of both functions COMBINED instead of the maximum of the two. Even with the normal 8Kb stacks this is a danger since we see some functions grow 3Kb to 4Kb of stack use this way. With 4Kb stacks, 4Kb of stack usage growth obviously is deadly ;-( but even with 8Kb stacks it's pure lottery. Disabling -funit-at-a-time also exposes another thing in the -mm tree; the attribute always_inline is considered harmful by gcc folks in that when gcc makes a decision to NOT inline a function marked this way, it throws an error. Disabling -funit-at-a-time disables some of the agressive inlining (eg of large functions that come later in the .c file) so this would make your tree not compile. --- 25-akpm/arch/i386/Kconfig | 9 + 25-akpm/arch/i386/Makefile | 6 - 25-akpm/arch/i386/kernel/i8259.c | 3 25-akpm/arch/i386/kernel/irq.c | 150 ++++++++++++++++++++++++++++++++- 25-akpm/arch/i386/kernel/smpboot.c | 2 25-akpm/arch/i386/kernel/traps.c | 18 ++- 25-akpm/include/asm-i386/irq.h | 20 ++++ 25-akpm/include/asm-i386/thread_info.h | 24 ++++- 25-akpm/include/linux/compiler-gcc3.h | 2 25-akpm/kernel/softirq.c | 70 +++++++++------ 10 files changed, 262 insertions(+), 42 deletions(-) diff -puN arch/i386/Kconfig~ia32-4k-stacks arch/i386/Kconfig --- 25/arch/i386/Kconfig~ia32-4k-stacks 2004-03-23 23:00:13.748845000 -0800 +++ 25-akpm/arch/i386/Kconfig 2004-03-23 23:00:13.763842720 -0800 @@ -1501,6 +1501,15 @@ config MAGIC_SYSRQ depends on KGDB_SYSRQ default y +config 4KSTACKS + bool "Use 4Kb for kernel stacks instead of 8Kb" + help + If you say Y here the kernel will use a 4Kb stacksize for the + kernel stack attached to each process/thread. This facilitates + running more threads on a system and also reduces the pressure + on the VM subsystem for higher order allocations. This option + will also use IRQ stacks to compensate for the reduced stackspace. + config X86_FIND_SMP_CONFIG bool depends on X86_LOCAL_APIC || X86_VOYAGER diff -puN arch/i386/kernel/i8259.c~ia32-4k-stacks arch/i386/kernel/i8259.c --- 25/arch/i386/kernel/i8259.c~ia32-4k-stacks 2004-03-23 23:00:13.750844696 -0800 +++ 25-akpm/arch/i386/kernel/i8259.c 2004-03-23 23:00:13.763842720 -0800 @@ -444,4 +444,7 @@ void __init init_IRQ(void) */ if (boot_cpu_data.hard_math && !cpu_has_fpu) setup_irq(FPU_IRQ, &fpu_irq); + + current_thread_info()->cpu = 0; + irq_ctx_init(0); } diff -puN arch/i386/kernel/irq.c~ia32-4k-stacks arch/i386/kernel/irq.c --- 25/arch/i386/kernel/irq.c~ia32-4k-stacks 2004-03-23 23:00:13.751844544 -0800 +++ 25-akpm/arch/i386/kernel/irq.c 2004-03-23 23:00:13.765842416 -0800 @@ -75,6 +75,14 @@ irq_desc_t irq_desc[NR_IRQS] __cacheline static void register_irq_proc (unsigned int irq); /* + * per-CPU IRQ handling stacks + */ +#ifdef CONFIG_4KSTACKS +union irq_ctx *hardirq_ctx[NR_CPUS]; +union irq_ctx *softirq_ctx[NR_CPUS]; +#endif + +/* * Special irq handlers. */ @@ -209,7 +217,7 @@ inline void synchronize_irq(unsigned int * waste of time and is not what some drivers would * prefer. */ -int handle_IRQ_event(unsigned int irq, +asmlinkage int handle_IRQ_event(unsigned int irq, struct pt_regs *regs, struct irqaction *action) { int status = 1; /* Force the "do bottom halves" bit */ @@ -432,7 +440,7 @@ asmlinkage unsigned int do_IRQ(struct pt __asm__ __volatile__("andl %%esp,%0" : "=r" (esp) : "0" (THREAD_SIZE - 1)); - if (unlikely(esp < (sizeof(struct thread_info) + 1024))) { + if (unlikely(esp < (sizeof(struct thread_info) + STACK_WARN))) { printk("do_IRQ: stack overflow: %ld\n", esp - sizeof(struct thread_info)); dump_stack(); @@ -480,11 +488,70 @@ asmlinkage unsigned int do_IRQ(struct pt * useful for irq hardware that does not mask cleanly in an * SMP environment. */ +#ifdef CONFIG_4KSTACKS + + for (;;) { + irqreturn_t action_ret; + u32 *isp; + union irq_ctx * curctx; + union irq_ctx * irqctx; + + curctx = (union irq_ctx *) current_thread_info(); + irqctx = hardirq_ctx[smp_processor_id()]; + + spin_unlock(&desc->lock); + + /* + * this is where we switch to the IRQ stack. However, if we are already using + * the IRQ stack (because we interrupted a hardirq handler) we can't do that + * and just have to keep using the current stack (which is the irq stack already + * after all) + */ + + if (curctx == irqctx) + action_ret = handle_IRQ_event(irq, ®s, action); + else { + /* build the stack frame on the IRQ stack */ + isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); + irqctx->tinfo.task = curctx->tinfo.task; + irqctx->tinfo.real_stack = curctx->tinfo.real_stack; + irqctx->tinfo.virtual_stack = curctx->tinfo.virtual_stack; + irqctx->tinfo.previous_esp = current_stack_pointer(); + + *--isp = (u32) action; + *--isp = (u32) ®s; + *--isp = (u32) irq; + + asm volatile( + " xchgl %%ebx,%%esp \n" + " call handle_IRQ_event \n" + " xchgl %%ebx,%%esp \n" + : "=a"(action_ret) + : "b"(isp) + : "memory", "cc", "edx", "ecx" + ); + + + } + spin_lock(&desc->lock); + if (!noirqdebug) + note_interrupt(irq, desc, action_ret); + if (curctx != irqctx) + irqctx->tinfo.task = NULL; + if (likely(!(desc->status & IRQ_PENDING))) + break; + desc->status &= ~IRQ_PENDING; + } + +#else + for (;;) { irqreturn_t action_ret; spin_unlock(&desc->lock); + action_ret = handle_IRQ_event(irq, ®s, action); + spin_lock(&desc->lock); if (!noirqdebug) note_interrupt(irq, desc, action_ret); @@ -492,6 +559,7 @@ asmlinkage unsigned int do_IRQ(struct pt break; desc->status &= ~IRQ_PENDING; } +#endif desc->status &= ~IRQ_INPROGRESS; out: @@ -1051,3 +1119,81 @@ void init_irq_proc (void) register_irq_proc(i); } + +#ifdef CONFIG_4KSTACKS +static char softirq_stack[NR_CPUS * THREAD_SIZE] __attribute__((__aligned__(THREAD_SIZE))); +static char hardirq_stack[NR_CPUS * THREAD_SIZE] __attribute__((__aligned__(THREAD_SIZE))); + +/* + * allocate per-cpu stacks for hardirq and for softirq processing + */ +void irq_ctx_init(int cpu) +{ + union irq_ctx *irqctx; + + if (hardirq_ctx[cpu]) + return; + + irqctx = (union irq_ctx*) &hardirq_stack[cpu*THREAD_SIZE]; + irqctx->tinfo.task = NULL; + irqctx->tinfo.exec_domain = NULL; + irqctx->tinfo.cpu = cpu; + irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; + irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); + + hardirq_ctx[cpu] = irqctx; + + irqctx = (union irq_ctx*) &softirq_stack[cpu*THREAD_SIZE]; + irqctx->tinfo.task = NULL; + irqctx->tinfo.exec_domain = NULL; + irqctx->tinfo.cpu = cpu; + irqctx->tinfo.preempt_count = SOFTIRQ_OFFSET; + irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); + + softirq_ctx[cpu] = irqctx; + + printk("CPU %u irqstacks, hard=%p soft=%p\n", + cpu,hardirq_ctx[cpu],softirq_ctx[cpu]); +} + +extern asmlinkage void __do_softirq(void); + +asmlinkage void do_softirq(void) +{ + unsigned long flags; + struct thread_info *curctx; + union irq_ctx *irqctx; + u32 *isp; + + if (in_interrupt()) + return; + + local_irq_save(flags); + + if (local_softirq_pending()) { + curctx = current_thread_info(); + irqctx = softirq_ctx[smp_processor_id()]; + irqctx->tinfo.task = curctx->task; + irqctx->tinfo.real_stack = curctx->real_stack; + irqctx->tinfo.virtual_stack = curctx->virtual_stack; + irqctx->tinfo.previous_esp = current_stack_pointer(); + + /* build the stack frame on the softirq stack */ + isp = (u32*) ((char*)irqctx + sizeof(*irqctx)); + + + asm volatile( + " xchgl %%ebx,%%esp \n" + " call __do_softirq \n" + " movl %%ebx,%%esp \n" + : "=b"(isp) + : "0"(isp) + : "memory", "cc", "edx", "ecx", "eax" + ); + } + + local_irq_restore(flags); +} + +EXPORT_SYMBOL(do_softirq); +#endif diff -puN arch/i386/kernel/smpboot.c~ia32-4k-stacks arch/i386/kernel/smpboot.c --- 25/arch/i386/kernel/smpboot.c~ia32-4k-stacks 2004-03-23 23:00:13.752844392 -0800 +++ 25-akpm/arch/i386/kernel/smpboot.c 2004-03-23 23:00:13.766842264 -0800 @@ -816,6 +816,8 @@ static int __init do_boot_cpu(int apicid /* Stack for startup_32 can be just as for start_secondary onwards */ stack_start.esp = (void *) idle->thread.esp; + irq_ctx_init(cpu); + /* * This grunge runs the startup process for * the targeted processor. diff -puN arch/i386/kernel/traps.c~ia32-4k-stacks arch/i386/kernel/traps.c --- 25/arch/i386/kernel/traps.c~ia32-4k-stacks 2004-03-23 23:00:13.754844088 -0800 +++ 25-akpm/arch/i386/kernel/traps.c 2004-03-23 23:00:13.767842112 -0800 @@ -139,12 +139,20 @@ void show_trace(struct task_struct *task #ifdef CONFIG_KALLSYMS printk("\n"); #endif - while (!kstack_end(stack)) { - addr = *stack++; - if (kernel_text_address(addr)) { - printk(" [<%08lx>] ", addr); - print_symbol("%s\n", addr); + while (1) { + struct thread_info *context; + context = (struct thread_info*) ((unsigned long)stack & (~(THREAD_SIZE - 1))); + while (!kstack_end(stack)) { + addr = *stack++; + if (kernel_text_address(addr)) { + printk(" [<%08lx>] ", addr); + print_symbol("%s\n", addr); + } } + stack = (unsigned long*)context->previous_esp; + if (!stack) + break; + printk(" =======================\n"); } printk("\n"); } diff -puN arch/i386/Makefile~ia32-4k-stacks arch/i386/Makefile --- 25/arch/i386/Makefile~ia32-4k-stacks 2004-03-23 23:00:13.755843936 -0800 +++ 25-akpm/arch/i386/Makefile 2004-03-23 23:00:13.767842112 -0800 @@ -56,9 +56,9 @@ cflags-$(CONFIG_X86_ELAN) += -march=i486 GCC_VERSION := $(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-version.sh $(CC)) cflags-$(CONFIG_REGPARM) += $(shell if [ $(GCC_VERSION) -ge 0300 ] ; then echo "-mregparm=3"; fi ;) -# Enable unit-at-a-time mode when possible. It shrinks the -# kernel considerably. -CFLAGS += $(call check_gcc,-funit-at-a-time,) +# Disable unit-at-a-time mode, it makes gcc use a lot more stack +# due to the lack of sharing of stacklots. +CFLAGS += $(call check_gcc,-fno-unit-at-a-time,) CFLAGS += $(cflags-y) diff -puN include/asm-i386/irq.h~ia32-4k-stacks include/asm-i386/irq.h --- 25/include/asm-i386/irq.h~ia32-4k-stacks 2004-03-23 23:00:13.756843784 -0800 +++ 25-akpm/include/asm-i386/irq.h 2004-03-23 23:00:13.768841960 -0800 @@ -14,6 +14,7 @@ #include /* include comes from machine specific directory */ #include "irq_vectors.h" +#include static __inline__ int irq_canonicalize(int irq) { @@ -30,4 +31,23 @@ extern int can_request_irq(unsigned int, #define ARCH_HAS_NMI_WATCHDOG /* See include/linux/nmi.h */ #endif +#ifdef CONFIG_4KSTACKS +/* + * per-CPU IRQ handling contexts (thread information and stack) + */ +union irq_ctx { + struct thread_info tinfo; + u32 stack[THREAD_SIZE/sizeof(u32)]; +}; + +extern union irq_ctx *hardirq_ctx[NR_CPUS]; +extern union irq_ctx *softirq_ctx[NR_CPUS]; + +extern void irq_ctx_init(int cpu); + +#define __ARCH_HAS_DO_SOFTIRQ +#else +#define irq_ctx_init(cpu) do { ; } while (0) +#endif + #endif /* _ASM_IRQ_H */ diff -puN include/asm-i386/thread_info.h~ia32-4k-stacks include/asm-i386/thread_info.h --- 25/include/asm-i386/thread_info.h~ia32-4k-stacks 2004-03-23 23:00:13.757843632 -0800 +++ 25-akpm/include/asm-i386/thread_info.h 2004-03-23 23:00:13.768841960 -0800 @@ -9,6 +9,9 @@ #ifdef __KERNEL__ +#include +#include + #ifndef __ASSEMBLY__ #include #endif @@ -29,6 +32,10 @@ struct thread_info { __u32 cpu; /* current CPU */ __s32 preempt_count; /* 0 => preemptable, <0 => BUG */ + unsigned long previous_esp; /* ESP of the previous stack in case + of nested (IRQ) stacks + */ + mm_segment_t addr_limit; /* thread address space: 0-0xBFFFFFFF for user-thead 0-0xFFFFFFFF for kernel-thread @@ -53,7 +60,13 @@ struct thread_info { #endif #define PREEMPT_ACTIVE 0x4000000 +#ifdef CONFIG_4KSTACKS +#define THREAD_SIZE (4096) +#else +#define THREAD_SIZE (8192) +#endif +#define STACK_WARN (THREAD_SIZE/8) /* * macros/functions for gaining access to the thread information structure * @@ -77,7 +90,6 @@ struct thread_info { #define init_thread_info (init_thread_union.thread_info) #define init_stack (init_thread_union.stack) -#define THREAD_SIZE (2*PAGE_SIZE) /* how to get the thread information struct from C */ static inline struct thread_info *current_thread_info(void) @@ -87,6 +99,14 @@ static inline struct thread_info *curren return ti; } +/* how to get the current stack pointer from C */ +static inline unsigned long current_stack_pointer(void) +{ + unsigned long ti; + __asm__("movl %%esp,%0; ":"=r" (ti) : ); + return ti; +} + /* thread information allocation */ #ifdef CONFIG_DEBUG_STACK_USAGE #define alloc_thread_info(tsk) \ @@ -108,8 +128,6 @@ static inline struct thread_info *curren #else /* !__ASSEMBLY__ */ -#define THREAD_SIZE 8192 - /* how to get the thread information struct from ASM */ #define GET_THREAD_INFO(reg) \ movl $-THREAD_SIZE, reg; \ diff -puN include/linux/compiler-gcc3.h~ia32-4k-stacks include/linux/compiler-gcc3.h --- 25/include/linux/compiler-gcc3.h~ia32-4k-stacks 2004-03-23 23:00:13.759843328 -0800 +++ 25-akpm/include/linux/compiler-gcc3.h 2004-03-23 23:00:13.769841808 -0800 @@ -3,7 +3,7 @@ /* These definitions are for GCC v3.x. */ #include -#if __GNUC_MINOR__ >= 1 +#if __GNUC_MINOR__ >= 1 && __GNUC_MINOR__ < 4 # define inline __inline__ __attribute__((always_inline)) # define __inline__ __inline__ __attribute__((always_inline)) # define __inline __inline__ __attribute__((always_inline)) diff -puN kernel/softirq.c~ia32-4k-stacks kernel/softirq.c --- 25/kernel/softirq.c~ia32-4k-stacks 2004-03-23 23:00:13.760843176 -0800 +++ 25-akpm/kernel/softirq.c 2004-03-23 23:00:13.769841808 -0800 @@ -16,6 +16,7 @@ #include #include +#include /* - No shared variables, all the data are CPU local. - If a softirq needs serialization, let it serialize itself @@ -69,53 +70,66 @@ static inline void wakeup_softirqd(void) */ #define MAX_SOFTIRQ_RESTART 10 -asmlinkage void do_softirq(void) +asmlinkage void __do_softirq(void) { - int max_restart = MAX_SOFTIRQ_RESTART; + struct softirq_action *h; __u32 pending; - unsigned long flags; + int max_restart = MAX_SOFTIRQ_RESTART; - if (in_interrupt()) - return; + pending = local_softirq_pending(); - local_irq_save(flags); + local_bh_disable(); +restart: + /* Reset the pending bitmask before enabling irqs */ + local_softirq_pending() = 0; + + local_irq_enable(); + + h = softirq_vec; + + do { + if (pending & 1) + h->action(h); + h++; + pending >>= 1; + } while (pending); + + local_irq_disable(); pending = local_softirq_pending(); + if (pending && --max_restart) + goto restart; - if (pending) { - struct softirq_action *h; + if (pending) + wakeup_softirqd(); - local_bh_disable(); -restart: - /* Reset the pending bitmask before enabling irqs */ - local_softirq_pending() = 0; + __local_bh_enable(); +} - local_irq_enable(); +#ifndef __ARCH_HAS_DO_SOFTIRQ + +asmlinkage void do_softirq(void) +{ + __u32 pending; + unsigned long flags; - h = softirq_vec; + if (in_interrupt()) + return; - do { - if (pending & 1) - h->action(h); - h++; - pending >>= 1; - } while (pending); + local_irq_save(flags); - local_irq_disable(); + pending = local_softirq_pending(); - pending = local_softirq_pending(); - if (pending && --max_restart) - goto restart; - if (pending) - wakeup_softirqd(); - __local_bh_enable(); - } + if (pending) + __do_softirq(); local_irq_restore(flags); } EXPORT_SYMBOL(do_softirq); +#endif + void local_bh_enable(void) { if (in_irq()) { _