From: Nick Piggin <nickpiggin@yahoo.com.au>

Add some likely/unliklies, a for_each_cpu => for_each_cpu_online, and close
the sched_exit race.

Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---

 25-akpm/kernel/sched.c |   20 +++++++++-----------
 1 files changed, 9 insertions(+), 11 deletions(-)

diff -puN kernel/sched.c~sched-misc kernel/sched.c
--- 25/kernel/sched.c~sched-misc	2004-07-13 13:17:29.783906688 -0700
+++ 25-akpm/kernel/sched.c	2004-07-13 13:17:29.789905776 -0700
@@ -942,7 +942,7 @@ void fastcall sched_fork(task_t *p, unsi
 	p->first_time_slice = 1;
 	current->time_slice >>= 1;
 	p->timestamp = sched_clock();
-	if (!current->time_slice) {
+	if (unlikely(!current->time_slice)) {
 		/*
 		 * This case is rare, it happens when the parent has only
 		 * a single jiffy left from its timeslice. Taking the
@@ -1049,18 +1049,16 @@ void fastcall sched_exit(task_t * p)
 	unsigned long flags;
 	runqueue_t *rq;
 
-	local_irq_save(flags);
-	if (p->first_time_slice) {
-		p->parent->time_slice += p->time_slice;
-		if (unlikely(p->parent->time_slice > MAX_TIMESLICE))
-			p->parent->time_slice = MAX_TIMESLICE;
-	}
-	local_irq_restore(flags);
 	/*
 	 * If the child was a (relative-) CPU hog then decrease
 	 * the sleep_avg of the parent as well.
 	 */
 	rq = task_rq_lock(p->parent, &flags);
+	if (p->first_time_slice) {
+		p->parent->time_slice += p->time_slice;
+		if (unlikely(p->parent->time_slice > MAX_TIMESLICE))
+			p->parent->time_slice = MAX_TIMESLICE;
+	}
 	if (p->sleep_avg < p->parent->sleep_avg)
 		p->parent->sleep_avg = p->parent->sleep_avg /
 		(EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg /
@@ -1160,7 +1158,7 @@ unsigned long nr_running(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_running;
 
 	return sum;
@@ -2273,7 +2271,7 @@ need_resched:
 		goto switch_tasks;
 	}
 
-	if (!rt_task(next) && next->activated > 0) {
+	if (rt_task(next) && next->activated > 0) {
 		unsigned long long delta = now - next->timestamp;
 
 		if (next->activated == 1)
@@ -2314,7 +2312,7 @@ switch_tasks:
 
 	reacquire_kernel_lock(current);
 	preempt_enable_no_resched();
-	if (test_thread_flag(TIF_NEED_RESCHED))
+	if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
 		goto need_resched;
 }
 
_