From: Nick Piggin Clean up init_idle to not use wake_up_forked_process, then undo all the stuff that call does. Instead, do everything in init_idle. Make double_rq_lock depend on CONFIG_SMP because it is no longer used on UP. Signed-off-by: Nick Piggin Signed-off-by: Andrew Morton --- 25-akpm/arch/alpha/kernel/smp.c | 2 25-akpm/arch/i386/kernel/smpboot.c | 7 -- 25-akpm/arch/i386/mach-voyager/voyager_smp.c | 5 + 25-akpm/arch/ia64/kernel/smpboot.c | 7 -- 25-akpm/arch/mips/kernel/smp.c | 8 -- 25-akpm/arch/parisc/kernel/smp.c | 1 25-akpm/arch/ppc/kernel/smp.c | 2 25-akpm/arch/ppc64/kernel/smp.c | 1 25-akpm/arch/s390/kernel/smp.c | 5 + 25-akpm/arch/sh/kernel/smp.c | 2 25-akpm/arch/x86_64/kernel/smpboot.c | 7 -- 25-akpm/init/main.c | 15 ++--- 25-akpm/kernel/sched.c | 77 +++++++++++++-------------- 13 files changed, 61 insertions(+), 78 deletions(-) diff -puN arch/alpha/kernel/smp.c~sched-clean-init-idle arch/alpha/kernel/smp.c --- 25/arch/alpha/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.116768224 -0700 +++ 25-akpm/arch/alpha/kernel/smp.c 2004-07-13 13:17:24.139764728 -0700 @@ -439,8 +439,6 @@ smp_boot_one_cpu(int cpuid) if (IS_ERR(idle)) panic("failed fork for CPU %d", cpuid); - wake_up_forked_process(idle); - init_idle(idle, cpuid); unhash_process(idle); diff -puN arch/i386/kernel/smpboot.c~sched-clean-init-idle arch/i386/kernel/smpboot.c --- 25/arch/i386/kernel/smpboot.c~sched-clean-init-idle 2004-07-13 13:17:24.118767920 -0700 +++ 25-akpm/arch/i386/kernel/smpboot.c 2004-07-13 13:17:24.140764576 -0700 @@ -800,16 +800,13 @@ static int __init do_boot_cpu(int apicid idle = fork_by_hand(); if (IS_ERR(idle)) panic("failed fork for CPU %d", cpu); - wake_up_forked_process(idle); - /* - * We remove it from the pidhash and the runqueue - * once we got the process: - */ + /* Make this the idle thread */ init_idle(idle, cpu); idle->thread.eip = (unsigned long) start_secondary; + /* Remove it from the pidhash */ unhash_process(idle); /* start_eip had better be page-aligned! */ diff -puN arch/i386/mach-voyager/voyager_smp.c~sched-clean-init-idle arch/i386/mach-voyager/voyager_smp.c --- 25/arch/i386/mach-voyager/voyager_smp.c~sched-clean-init-idle 2004-07-13 13:17:24.119767768 -0700 +++ 25-akpm/arch/i386/mach-voyager/voyager_smp.c 2004-07-13 13:17:24.142764272 -0700 @@ -591,11 +591,12 @@ do_boot_cpu(__u8 cpu) if(IS_ERR(idle)) panic("failed fork for CPU%d", cpu); - wake_up_forked_process(idle); - + /* Make this the idle thread */ init_idle(idle, cpu); idle->thread.eip = (unsigned long) start_secondary; + + /* Remove it from the pidhash */ unhash_process(idle); /* init_tasks (in sched.c) is indexed logically */ stack_start.esp = (void *) idle->thread.esp; diff -puN arch/ia64/kernel/smpboot.c~sched-clean-init-idle arch/ia64/kernel/smpboot.c --- 25/arch/ia64/kernel/smpboot.c~sched-clean-init-idle 2004-07-13 13:17:24.121767464 -0700 +++ 25-akpm/arch/ia64/kernel/smpboot.c 2004-07-13 13:17:24.143764120 -0700 @@ -400,14 +400,11 @@ do_boot_cpu (int sapicid, int cpu) if (IS_ERR(c_idle.idle)) panic("failed fork for CPU %d", cpu); - wake_up_forked_process(c_idle.idle); - /* - * We remove it from the pidhash and the runqueue - * once we got the process: - */ + /* Make this the idle thread */ init_idle(c_idle.idle, cpu); + /* Remove it from the pidhash */ unhash_process(c_idle.idle); task_for_booting_cpu = c_idle.idle; diff -puN arch/mips/kernel/smp.c~sched-clean-init-idle arch/mips/kernel/smp.c --- 25/arch/mips/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.123767160 -0700 +++ 25-akpm/arch/mips/kernel/smp.c 2004-07-13 13:17:24.143764120 -0700 @@ -279,14 +279,10 @@ static int __init do_boot_cpu(int cpu) if (IS_ERR(idle)) panic("failed fork for CPU %d\n", cpu); - wake_up_forked_process(idle); - - /* - * We remove it from the pidhash and the runqueue once we've - * got the process: - */ + /* Make this the idle thread */ init_idle(idle, cpu); + /* Remove it from the pidhash */ unhash_process(idle); prom_boot_secondary(cpu, idle); diff -puN arch/parisc/kernel/smp.c~sched-clean-init-idle arch/parisc/kernel/smp.c --- 25/arch/parisc/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.124767008 -0700 +++ 25-akpm/arch/parisc/kernel/smp.c 2004-07-13 13:17:24.144763968 -0700 @@ -543,7 +543,6 @@ int __init smp_boot_one_cpu(int cpuid, i if (IS_ERR(idle)) panic("SMP: fork failed for CPU:%d", cpuid); - wake_up_forked_process(idle); init_idle(idle, cpunum); unhash_process(idle); idle->thread_info->cpu = cpunum; diff -puN arch/ppc64/kernel/smp.c~sched-clean-init-idle arch/ppc64/kernel/smp.c --- 25/arch/ppc64/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.126766704 -0700 +++ 25-akpm/arch/ppc64/kernel/smp.c 2004-07-13 13:17:24.145763816 -0700 @@ -804,7 +804,6 @@ static void __init smp_create_idle(unsig if (IS_ERR(p)) panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p)); - wake_up_forked_process(p); init_idle(p, cpu); unhash_process(p); diff -puN arch/ppc/kernel/smp.c~sched-clean-init-idle arch/ppc/kernel/smp.c --- 25/arch/ppc/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.127766552 -0700 +++ 25-akpm/arch/ppc/kernel/smp.c 2004-07-13 13:17:24.145763816 -0700 @@ -375,8 +375,6 @@ int __cpu_up(unsigned int cpu) p = copy_process(CLONE_VM|CLONE_IDLETASK, 0, ®s, 0, NULL, NULL); if (IS_ERR(p)) panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p)); - wake_up_forked_process(p); - init_idle(p, cpu); unhash_process(p); diff -puN arch/s390/kernel/smp.c~sched-clean-init-idle arch/s390/kernel/smp.c --- 25/arch/s390/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.129766248 -0700 +++ 25-akpm/arch/s390/kernel/smp.c 2004-07-13 13:17:24.146763664 -0700 @@ -574,9 +574,12 @@ static void __init smp_create_idle(unsig if (IS_ERR(p)) panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p)); - wake_up_forked_process(p); + /* Make this the idle thread */ init_idle(p, cpu); + + /* Remove it from the pidhash */ unhash_process(p); + current_set[cpu] = p; } diff -puN arch/sh/kernel/smp.c~sched-clean-init-idle arch/sh/kernel/smp.c --- 25/arch/sh/kernel/smp.c~sched-clean-init-idle 2004-07-13 13:17:24.130766096 -0700 +++ 25-akpm/arch/sh/kernel/smp.c 2004-07-13 13:17:24.146763664 -0700 @@ -106,8 +106,6 @@ int __cpu_up(unsigned int cpu) if (IS_ERR(tsk)) panic("Failed forking idle task for cpu %d\n", cpu); - wake_up_forked_process(tsk); - init_idle(tsk, cpu); unhash_process(tsk); diff -puN arch/x86_64/kernel/smpboot.c~sched-clean-init-idle arch/x86_64/kernel/smpboot.c --- 25/arch/x86_64/kernel/smpboot.c~sched-clean-init-idle 2004-07-13 13:17:24.132765792 -0700 +++ 25-akpm/arch/x86_64/kernel/smpboot.c 2004-07-13 13:17:24.147763512 -0700 @@ -576,15 +576,12 @@ static void __init do_boot_cpu (int apic idle = fork_by_hand(); if (IS_ERR(idle)) panic("failed fork for CPU %d", cpu); - wake_up_forked_process(idle); x86_cpu_to_apicid[cpu] = apicid; - /* - * We remove it from the pidhash and the runqueue - * once we got the process: - */ + /* Make this the idle thread */ init_idle(idle,cpu); + /* Remove it from the pidhash */ unhash_process(idle); cpu_pda[cpu].pcurrent = idle; diff -puN init/main.c~sched-clean-init-idle init/main.c --- 25/init/main.c~sched-clean-init-idle 2004-07-13 13:17:24.133765640 -0700 +++ 25-akpm/init/main.c 2004-07-13 13:17:24.148763360 -0700 @@ -466,6 +466,14 @@ asmlinkage void __init start_kernel(void */ sched_init(); + /* + * Make us the idle thread. Technically, schedule() should not be + * called from this thread, however somewhere below it might be, + * but because we are the idle thread, we just pick up running again + * when this runqueue becomes "idle". + */ + init_idle(current, smp_processor_id()); + build_all_zonelists(); page_alloc_init(); trap_init(); @@ -530,13 +538,6 @@ asmlinkage void __init start_kernel(void #endif check_bugs(); - /* - * We count on the initial thread going ok - * Like idlers init is an unlocked kernel thread, which will - * make syscalls (and thus be locked). - */ - init_idle(current, smp_processor_id()); - /* Do the rest non-__init'ed, we're now alive */ rest_init(); } diff -puN kernel/sched.c~sched-clean-init-idle kernel/sched.c --- 25/kernel/sched.c~sched-clean-init-idle 2004-07-13 13:17:24.135765336 -0700 +++ 25-akpm/kernel/sched.c 2004-07-13 13:17:24.151762904 -0700 @@ -1122,6 +1122,15 @@ unsigned long nr_iowait(void) return sum; } +enum idle_type +{ + IDLE, + NOT_IDLE, + NEWLY_IDLE, +}; + +#ifdef CONFIG_SMP + /* * double_rq_lock - safely lock two runqueues * @@ -1156,14 +1165,20 @@ static void double_rq_unlock(runqueue_t spin_unlock(&rq2->lock); } -enum idle_type +/* + * double_lock_balance - lock the busiest runqueue, this_rq is locked already. + */ +static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest) { - IDLE, - NOT_IDLE, - NEWLY_IDLE, -}; - -#ifdef CONFIG_SMP + if (unlikely(!spin_trylock(&busiest->lock))) { + if (busiest < this_rq) { + spin_unlock(&this_rq->lock); + spin_lock(&busiest->lock); + spin_lock(&this_rq->lock); + } else + spin_lock(&busiest->lock); + } +} /* * find_idlest_cpu - find the least busy runqueue. @@ -1359,21 +1374,6 @@ out: } /* - * double_lock_balance - lock the busiest runqueue, this_rq is locked already. - */ -static void double_lock_balance(runqueue_t *this_rq, runqueue_t *busiest) -{ - if (unlikely(!spin_trylock(&busiest->lock))) { - if (busiest < this_rq) { - spin_unlock(&this_rq->lock); - spin_lock(&busiest->lock); - spin_lock(&this_rq->lock); - } else - spin_lock(&busiest->lock); - } -} - -/* * pull_task - move a task from a remote runqueue to the local runqueue. * Both runqueues must be locked. */ @@ -2210,6 +2210,15 @@ need_resched: prev = current; rq = this_rq(); + /* + * The idle thread is not allowed to schedule! + * Remove this check after it has been exercised a bit. + */ + if (unlikely(current == rq->idle) && current->state != TASK_RUNNING) { + printk(KERN_ERR "bad: scheduling from the idle thread!\n"); + dump_stack(); + } + release_kernel_lock(prev); now = sched_clock(); if (likely(now - prev->timestamp < NS_MAX_SLEEP_AVG)) @@ -3276,21 +3285,20 @@ void show_state(void) void __devinit init_idle(task_t *idle, int cpu) { - runqueue_t *idle_rq = cpu_rq(cpu), *rq = cpu_rq(task_cpu(idle)); + runqueue_t *rq = cpu_rq(cpu); unsigned long flags; - local_irq_save(flags); - double_rq_lock(idle_rq, rq); - - idle_rq->curr = idle_rq->idle = idle; - deactivate_task(idle, rq); + idle->sleep_avg = 0; + idle->interactive_credit = 0; idle->array = NULL; idle->prio = MAX_PRIO; idle->state = TASK_RUNNING; set_task_cpu(idle, cpu); - double_rq_unlock(idle_rq, rq); + + spin_lock_irqsave(&rq->lock, flags); + rq->curr = rq->idle = idle; set_tsk_need_resched(idle); - local_irq_restore(flags); + spin_unlock_irqrestore(&rq->lock, flags); /* Set the preempt count _outside_ the spinlocks! */ #ifdef CONFIG_PREEMPT @@ -3968,15 +3976,6 @@ void __init sched_init(void) __set_bit(MAX_PRIO, array->bitmap); } } - /* - * We have to do a little magic to get the first - * thread right in SMP mode. - */ - rq = this_rq(); - rq->curr = current; - rq->idle = current; - set_task_cpu(current, smp_processor_id()); - wake_up_forked_process(current); /* * The boot idle thread does lazy MMU switching as well: _