Move rwsem's up_read wakeups out of the semaphore's wait_lock linux-2.6-npiggin/lib/rwsem-spinlock.c | 27 ++++++++++------- linux-2.6-npiggin/lib/rwsem.c | 51 +++++++++++++++------------------ 2 files changed, 40 insertions(+), 38 deletions(-) diff -puN lib/rwsem.c~rwsem-scale lib/rwsem.c --- linux-2.6/lib/rwsem.c~rwsem-scale 2004-04-12 00:28:52.000000000 +1000 +++ linux-2.6-npiggin/lib/rwsem.c 2004-04-12 00:36:58.000000000 +1000 @@ -36,13 +36,15 @@ void rwsemtrace(struct rw_semaphore *sem * - the spinlock must be held by the caller * - woken process blocks are discarded from the list after having flags zeroised * - writers are only woken if wakewrite is non-zero + * + * The spinlock will be dropped by this function */ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) { + LIST_HEAD(wake_list); struct rwsem_waiter *waiter; - struct list_head *next; signed long oldcount; - int woken, loop; + int woken; rwsemtrace(sem,"Entering __rwsem_do_wake"); @@ -64,9 +66,8 @@ static inline struct rw_semaphore *__rws if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE)) goto readers_only; - list_del(&waiter->list); + list_move_tail(&waiter->list, &wake_list); waiter->flags = 0; - wake_up_process(waiter->task); goto out; /* don't want to wake any writers */ @@ -75,39 +76,36 @@ static inline struct rw_semaphore *__rws if (waiter->flags & RWSEM_WAITING_FOR_WRITE) goto out; - /* grant an infinite number of read locks to the readers at the front of the queue - * - note we increment the 'active part' of the count by the number of readers (less one - * for the activity decrement we've already done) before waking any processes up + /* grant an infinite number of read locks to the readers at the front + * of the queue - note we increment the 'active part' of the count by + * the number of readers (less one for the activity decrement we've + * already done) before waking any processes up */ readers_only: woken = 0; do { + list_move_tail(&waiter->list, &wake_list); + waiter->flags = 0; woken++; if (waiter->list.next==&sem->wait_list) break; - waiter = list_entry(waiter->list.next,struct rwsem_waiter,list); + waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); } while (waiter->flags & RWSEM_WAITING_FOR_READ); - loop = woken; woken *= RWSEM_ACTIVE_BIAS-RWSEM_WAITING_BIAS; woken -= RWSEM_ACTIVE_BIAS; rwsem_atomic_add(woken,sem); - next = sem->wait_list.next; - for (; loop>0; loop--) { - waiter = list_entry(next,struct rwsem_waiter,list); - next = waiter->list.next; - waiter->flags = 0; + out: + spin_unlock(&sem->wait_lock); + while (!list_empty(&wake_list)) { + waiter = list_entry(wake_list.next,struct rwsem_waiter,list); + list_del(&waiter->list); wake_up_process(waiter->task); } - - sem->wait_list.next = next; - next->prev = &sem->wait_list; - - out: rwsemtrace(sem,"Leaving __rwsem_do_wake"); return sem; @@ -131,9 +129,8 @@ static inline struct rw_semaphore *rwsem set_task_state(tsk,TASK_UNINTERRUPTIBLE); /* set up my own style of waitqueue */ - spin_lock(&sem->wait_lock); waiter->task = tsk; - + spin_lock(&sem->wait_lock); list_add_tail(&waiter->list,&sem->wait_list); /* note that we're now waiting on the lock, but no longer actively read-locking */ @@ -144,8 +141,8 @@ static inline struct rw_semaphore *rwsem */ if (!(count & RWSEM_ACTIVE_MASK)) sem = __rwsem_do_wake(sem,1); - - spin_unlock(&sem->wait_lock); + else + spin_unlock(&sem->wait_lock); /* wait to be given the lock */ for (;;) { @@ -205,8 +202,8 @@ struct rw_semaphore fastcall *rwsem_wake /* do nothing if list empty */ if (!list_empty(&sem->wait_list)) sem = __rwsem_do_wake(sem,1); - - spin_unlock(&sem->wait_lock); + else + spin_unlock(&sem->wait_lock); rwsemtrace(sem,"Leaving rwsem_wake"); @@ -227,8 +224,8 @@ struct rw_semaphore fastcall *rwsem_down /* do nothing if list empty */ if (!list_empty(&sem->wait_list)) sem = __rwsem_do_wake(sem,0); - - spin_unlock(&sem->wait_lock); + else + spin_unlock(&sem->wait_lock); rwsemtrace(sem,"Leaving rwsem_downgrade_wake"); return sem; diff -puN lib/rwsem-spinlock.c~rwsem-scale lib/rwsem-spinlock.c --- linux-2.6/lib/rwsem-spinlock.c~rwsem-scale 2004-04-12 00:28:52.000000000 +1000 +++ linux-2.6-npiggin/lib/rwsem-spinlock.c 2004-04-12 11:09:54.000000000 +1000 @@ -47,9 +47,12 @@ void fastcall init_rwsem(struct rw_semap * - the spinlock must be held by the caller * - woken process blocks are discarded from the list after having flags zeroised * - writers are only woken if wakewrite is non-zero + * + * The spinlock will be dropped by this function */ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int wakewrite) { + LIST_HEAD(wake_list); struct rwsem_waiter *waiter; int woken; @@ -69,9 +72,8 @@ static inline struct rw_semaphore *__rws */ if (waiter->flags & RWSEM_WAITING_FOR_WRITE) { sem->activity = -1; - list_del(&waiter->list); + list_move_tail(&waiter->list, &wake_list); waiter->flags = 0; - wake_up_process(waiter->task); goto out; } @@ -79,20 +81,23 @@ static inline struct rw_semaphore *__rws dont_wake_writers: woken = 0; while (waiter->flags&RWSEM_WAITING_FOR_READ) { - struct list_head *next = waiter->list.next; - - list_del(&waiter->list); + list_move_tail(&waiter->list, &wake_list); waiter->flags = 0; - wake_up_process(waiter->task); woken++; if (list_empty(&sem->wait_list)) break; - waiter = list_entry(next,struct rwsem_waiter,list); + waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); } sem->activity += woken; out: + spin_unlock(&sem->wait_lock); + while (!list_empty(&wake_list)) { + waiter = list_entry(wake_list.next, struct rwsem_waiter, list); + list_del(&waiter->list); + wake_up_process(waiter->task); + } rwsemtrace(sem,"Leaving __rwsem_do_wake"); return sem; } @@ -278,8 +283,8 @@ void fastcall __up_write(struct rw_semap sem->activity = 0; if (!list_empty(&sem->wait_list)) sem = __rwsem_do_wake(sem, 1); - - spin_unlock(&sem->wait_lock); + else + spin_unlock(&sem->wait_lock); rwsemtrace(sem,"Leaving __up_write"); } @@ -297,8 +302,8 @@ void fastcall __downgrade_write(struct r sem->activity = 1; if (!list_empty(&sem->wait_list)) sem = __rwsem_do_wake(sem,0); - - spin_unlock(&sem->wait_lock); + else + spin_unlock(&sem->wait_lock); rwsemtrace(sem,"Leaving __downgrade_write"); } _