diff -urNp 2.4.19rc2/arch/alpha/config.in 2.4.19rc2aa1/arch/alpha/config.in --- 2.4.19rc2/arch/alpha/config.in Wed Jul 17 13:28:23 2002 +++ 2.4.19rc2aa1/arch/alpha/config.in Wed Jul 17 14:28:57 2002 @@ -5,8 +5,6 @@ define_bool CONFIG_ALPHA y define_bool CONFIG_UID16 n -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y mainmenu_name "Kernel configuration of Linux for Alpha machines" diff -urNp 2.4.19rc2/arch/arm/config.in 2.4.19rc2aa1/arch/arm/config.in --- 2.4.19rc2/arch/arm/config.in Wed Jul 17 13:28:23 2002 +++ 2.4.19rc2aa1/arch/arm/config.in Wed Jul 17 14:28:57 2002 @@ -9,8 +9,6 @@ define_bool CONFIG_EISA n define_bool CONFIG_SBUS n define_bool CONFIG_MCA n define_bool CONFIG_UID16 y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n define_bool CONFIG_GENERIC_BUST_SPINLOCK n define_bool CONFIG_GENERIC_ISA_DMA n diff -urNp 2.4.19rc2/arch/cris/config.in 2.4.19rc2aa1/arch/cris/config.in --- 2.4.19rc2/arch/cris/config.in Mon Feb 25 22:05:04 2002 +++ 2.4.19rc2aa1/arch/cris/config.in Wed Jul 17 14:28:57 2002 @@ -5,8 +5,6 @@ mainmenu_name "Linux/CRIS Kernel Configuration" define_bool CONFIG_UID16 y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n mainmenu_option next_comment comment 'Code maturity level options' diff -urNp 2.4.19rc2/arch/i386/config.in 2.4.19rc2aa1/arch/i386/config.in --- 2.4.19rc2/arch/i386/config.in Wed Jul 17 13:28:23 2002 +++ 2.4.19rc2aa1/arch/i386/config.in Wed Jul 17 14:28:57 2002 @@ -51,8 +51,6 @@ if [ "$CONFIG_M386" = "y" ]; then define_bool CONFIG_X86_CMPXCHG n define_bool CONFIG_X86_XADD n define_int CONFIG_X86_L1_CACHE_SHIFT 4 - define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y - define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n define_bool CONFIG_X86_PPRO_FENCE y else define_bool CONFIG_X86_WP_WORKS_OK y @@ -61,8 +59,6 @@ else define_bool CONFIG_X86_XADD y define_bool CONFIG_X86_BSWAP y define_bool CONFIG_X86_POPAD_OK y - define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n - define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y fi if [ "$CONFIG_M486" = "y" ]; then define_int CONFIG_X86_L1_CACHE_SHIFT 4 diff -urNp 2.4.19rc2/arch/ia64/config.in 2.4.19rc2aa1/arch/ia64/config.in --- 2.4.19rc2/arch/ia64/config.in Wed Jul 17 13:28:29 2002 +++ 2.4.19rc2aa1/arch/ia64/config.in Wed Jul 17 14:28:57 2002 @@ -23,8 +23,6 @@ define_bool CONFIG_ISA n define_bool CONFIG_EISA n define_bool CONFIG_MCA n define_bool CONFIG_SBUS n -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n if [ "$CONFIG_IA64_HP_SIM" = "n" ]; then define_bool CONFIG_ACPI y diff -urNp 2.4.19rc2/arch/m68k/config.in 2.4.19rc2aa1/arch/m68k/config.in --- 2.4.19rc2/arch/m68k/config.in Wed Jul 17 13:28:29 2002 +++ 2.4.19rc2aa1/arch/m68k/config.in Wed Jul 17 14:28:57 2002 @@ -4,8 +4,6 @@ # define_bool CONFIG_UID16 y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n mainmenu_name "Linux/68k Kernel Configuration" diff -urNp 2.4.19rc2/arch/mips/config.in 2.4.19rc2aa1/arch/mips/config.in --- 2.4.19rc2/arch/mips/config.in Wed Jul 17 13:28:29 2002 +++ 2.4.19rc2aa1/arch/mips/config.in Wed Jul 17 14:29:33 2002 @@ -84,9 +84,6 @@ bool 'Support for Toshiba JMR-TX3927 boa dep_bool 'High Memory Support (EXPERIMENTAL)' CONFIG_HIGHMEM $CONFIG_EXPERIMENTAL -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n - # # Select some configuration options automatically based on user selections. # diff -urNp 2.4.19rc2/arch/mips64/config.in 2.4.19rc2aa1/arch/mips64/config.in --- 2.4.19rc2/arch/mips64/config.in Wed Jul 17 13:28:30 2002 +++ 2.4.19rc2aa1/arch/mips64/config.in Wed Jul 17 14:29:44 2002 @@ -53,9 +53,6 @@ if [ "$CONFIG_SIBYTE_SB1250" = "y" ]; th fi fi -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n - # # Select some configuration options automatically based on user selections. # diff -urNp 2.4.19rc2/arch/parisc/config.in 2.4.19rc2aa1/arch/parisc/config.in --- 2.4.19rc2/arch/parisc/config.in Tue May 1 19:35:20 2001 +++ 2.4.19rc2aa1/arch/parisc/config.in Wed Jul 17 14:28:57 2002 @@ -7,8 +7,6 @@ mainmenu_name "Linux Kernel Configuratio define_bool CONFIG_PARISC y define_bool CONFIG_UID16 n -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n mainmenu_option next_comment comment 'Code maturity level options' diff -urNp 2.4.19rc2/arch/ppc/config.in 2.4.19rc2aa1/arch/ppc/config.in --- 2.4.19rc2/arch/ppc/config.in Wed Jul 17 13:28:30 2002 +++ 2.4.19rc2aa1/arch/ppc/config.in Wed Jul 17 14:28:57 2002 @@ -4,8 +4,6 @@ # see Documentation/kbuild/config-language.txt. # define_bool CONFIG_UID16 n -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y define_bool CONFIG_HAVE_DEC_LOCK y mainmenu_name "Linux/PowerPC Kernel Configuration" diff -urNp 2.4.19rc2/arch/s390/config.in 2.4.19rc2aa1/arch/s390/config.in --- 2.4.19rc2/arch/s390/config.in Wed Jul 17 13:28:31 2002 +++ 2.4.19rc2aa1/arch/s390/config.in Wed Jul 17 14:28:57 2002 @@ -7,8 +7,6 @@ define_bool CONFIG_ISA n define_bool CONFIG_EISA n define_bool CONFIG_MCA n define_bool CONFIG_UID16 y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y define_bool CONFIG_GENERIC_BUST_SPINLOCK n mainmenu_name "Linux Kernel Configuration" diff -urNp 2.4.19rc2/arch/s390x/config.in 2.4.19rc2aa1/arch/s390x/config.in --- 2.4.19rc2/arch/s390x/config.in Wed Jul 17 13:28:31 2002 +++ 2.4.19rc2aa1/arch/s390x/config.in Wed Jul 17 14:28:57 2002 @@ -6,8 +6,6 @@ define_bool CONFIG_ISA n define_bool CONFIG_EISA n define_bool CONFIG_MCA n -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y define_bool CONFIG_GENERIC_BUST_SPINLOCK n mainmenu_name "Linux Kernel Configuration" diff -urNp 2.4.19rc2/arch/sh/config.in 2.4.19rc2aa1/arch/sh/config.in --- 2.4.19rc2/arch/sh/config.in Mon Feb 25 22:05:04 2002 +++ 2.4.19rc2aa1/arch/sh/config.in Wed Jul 17 14:28:57 2002 @@ -7,8 +7,6 @@ mainmenu_name "Linux/SuperH Kernel Confi define_bool CONFIG_SUPERH y define_bool CONFIG_UID16 y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n mainmenu_option next_comment comment 'Code maturity level options' diff -urNp 2.4.19rc2/arch/sparc/config.in 2.4.19rc2aa1/arch/sparc/config.in --- 2.4.19rc2/arch/sparc/config.in Wed Jul 17 13:28:31 2002 +++ 2.4.19rc2aa1/arch/sparc/config.in Wed Jul 17 14:28:57 2002 @@ -48,8 +48,6 @@ define_bool CONFIG_SUN_KEYBOARD y define_bool CONFIG_SUN_CONSOLE y define_bool CONFIG_SUN_AUXIO y define_bool CONFIG_SUN_IO y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK y -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM n define_bool CONFIG_SUN_PM y bool 'Support for SUN4 machines (disables SUN4[CDM] support)' CONFIG_SUN4 diff -urNp 2.4.19rc2/arch/sparc64/config.in 2.4.19rc2aa1/arch/sparc64/config.in --- 2.4.19rc2/arch/sparc64/config.in Wed Jul 17 13:28:31 2002 +++ 2.4.19rc2aa1/arch/sparc64/config.in Wed Jul 17 14:28:57 2002 @@ -35,8 +35,6 @@ bool 'Support for hot-pluggable devices' # Global things across all Sun machines. define_bool CONFIG_HAVE_DEC_LOCK y -define_bool CONFIG_RWSEM_GENERIC_SPINLOCK n -define_bool CONFIG_RWSEM_XCHGADD_ALGORITHM y define_bool CONFIG_ISA n define_bool CONFIG_ISAPNP n define_bool CONFIG_EISA n diff -urNp 2.4.19rc2/include/asm-alpha/rwsem.h 2.4.19rc2aa1/include/asm-alpha/rwsem.h --- 2.4.19rc2/include/asm-alpha/rwsem.h Tue Jan 22 18:54:25 2002 +++ 2.4.19rc2aa1/include/asm-alpha/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,208 +0,0 @@ -#ifndef _ALPHA_RWSEM_H -#define _ALPHA_RWSEM_H - -/* - * Written by Ivan Kokshaysky , 2001. - * Based on asm-alpha/semaphore.h and asm-i386/rwsem.h - */ - -#ifndef _LINUX_RWSEM_H -#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead -#endif - -#ifdef __KERNEL__ - -#include -#include -#include - -struct rwsem_waiter; - -extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); - -/* - * the semaphore definition - */ -struct rw_semaphore { - long count; -#define RWSEM_UNLOCKED_VALUE 0x0000000000000000L -#define RWSEM_ACTIVE_BIAS 0x0000000000000001L -#define RWSEM_ACTIVE_MASK 0x00000000ffffffffL -#define RWSEM_WAITING_BIAS (-0x0000000100000000L) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - spinlock_t wait_lock; - struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif -}; - -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - -#define __RWSEM_INITIALIZER(name) \ - { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) __RWSEM_DEBUG_INIT } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif -} - -static inline void __down_read(struct rw_semaphore *sem) -{ - long oldcount; -#ifndef CONFIG_SMP - oldcount = sem->count; - sem->count += RWSEM_ACTIVE_READ_BIAS; -#else - long temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%3,%2\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - " mb\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp) - :"Ir" (RWSEM_ACTIVE_READ_BIAS), "m" (sem->count) : "memory"); -#endif - if (__builtin_expect(oldcount < 0, 0)) - rwsem_down_read_failed(sem); -} - -static inline void __down_write(struct rw_semaphore *sem) -{ - long oldcount; -#ifndef CONFIG_SMP - oldcount = sem->count; - sem->count += RWSEM_ACTIVE_WRITE_BIAS; -#else - long temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%3,%2\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - " mb\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp) - :"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory"); -#endif - if (__builtin_expect(oldcount, 0)) - rwsem_down_write_failed(sem); -} - -static inline void __up_read(struct rw_semaphore *sem) -{ - long oldcount; -#ifndef CONFIG_SMP - oldcount = sem->count; - sem->count -= RWSEM_ACTIVE_READ_BIAS; -#else - long temp; - __asm__ __volatile__( - " mb\n" - "1: ldq_l %0,%1\n" - " subq %0,%3,%2\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (oldcount), "=m" (sem->count), "=&r" (temp) - :"Ir" (RWSEM_ACTIVE_READ_BIAS), "m" (sem->count) : "memory"); -#endif - if (__builtin_expect(oldcount < 0, 0)) - if ((int)oldcount - RWSEM_ACTIVE_READ_BIAS == 0) - rwsem_wake(sem); -} - -static inline void __up_write(struct rw_semaphore *sem) -{ - long count; -#ifndef CONFIG_SMP - sem->count -= RWSEM_ACTIVE_WRITE_BIAS; - count = sem->count; -#else - long temp; - __asm__ __volatile__( - " mb\n" - "1: ldq_l %0,%1\n" - " subq %0,%3,%2\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - " subq %0,%3,%0\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (count), "=m" (sem->count), "=&r" (temp) - :"Ir" (RWSEM_ACTIVE_WRITE_BIAS), "m" (sem->count) : "memory"); -#endif - if (__builtin_expect(count, 0)) - if ((int)count == 0) - rwsem_wake(sem); -} - -static inline void rwsem_atomic_add(long val, struct rw_semaphore *sem) -{ -#ifndef CONFIG_SMP - sem->count += val; -#else - long temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%2,%0\n" - " stq_c %0,%1\n" - " beq %0,2f\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (temp), "=m" (sem->count) - :"Ir" (val), "m" (sem->count)); -#endif -} - -static inline long rwsem_atomic_update(long val, struct rw_semaphore *sem) -{ -#ifndef CONFIG_SMP - sem->count += val; - return sem->count; -#else - long ret, temp; - __asm__ __volatile__( - "1: ldq_l %0,%1\n" - " addq %0,%3,%2\n" - " addq %0,%3,%0\n" - " stq_c %2,%1\n" - " beq %2,2f\n" - ".subsection 2\n" - "2: br 1b\n" - ".previous" - :"=&r" (ret), "=m" (sem->count), "=&r" (temp) - :"Ir" (val), "m" (sem->count)); - - return ret; -#endif -} - -#endif /* __KERNEL__ */ -#endif /* _ALPHA_RWSEM_H */ diff -urNp 2.4.19rc2/include/asm-i386/rwsem.h 2.4.19rc2aa1/include/asm-i386/rwsem.h --- 2.4.19rc2/include/asm-i386/rwsem.h Wed Jul 17 13:28:49 2002 +++ 2.4.19rc2aa1/include/asm-i386/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,226 +0,0 @@ -/* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ - * - * Written by David Howells (dhowells@redhat.com). - * - * Derived from asm-i386/semaphore.h - * - * - * The MSW of the count is the negated number of active writers and waiting - * lockers, and the LSW is the total number of active locks - * - * The lock count is initialized to 0 (no active and no waiting lockers). - * - * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an - * uncontended lock. This can be determined because XADD returns the old value. - * Readers increment by 1 and see a positive value when uncontended, negative - * if there are writers (and maybe) readers waiting (in which case it goes to - * sleep). - * - * The value of WAITING_BIAS supports up to 32766 waiting processes. This can - * be extended to 65534 by manually checking the whole MSW rather than relying - * on the S flag. - * - * The value of ACTIVE_BIAS supports up to 65535 active processes. - * - * This should be totally fair - if anything is waiting, a process that wants a - * lock will go to the back of the queue. When the currently active lock is - * released, if there's a writer at the front of the queue, then that and only - * that will be woken up; if there's a bunch of consequtive readers at the - * front, then they'll all be woken up, but no other readers will be. - */ - -#ifndef _I386_RWSEM_H -#define _I386_RWSEM_H - -#ifndef _LINUX_RWSEM_H -#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead -#endif - -#ifdef __KERNEL__ - -#include -#include - -struct rwsem_waiter; - -extern struct rw_semaphore *FASTCALL(rwsem_down_read_failed(struct rw_semaphore *sem)); -extern struct rw_semaphore *FASTCALL(rwsem_down_write_failed(struct rw_semaphore *sem)); -extern struct rw_semaphore *FASTCALL(rwsem_wake(struct rw_semaphore *)); - -/* - * the semaphore definition - */ -struct rw_semaphore { - signed long count; -#define RWSEM_UNLOCKED_VALUE 0x00000000 -#define RWSEM_ACTIVE_BIAS 0x00000001 -#define RWSEM_ACTIVE_MASK 0x0000ffff -#define RWSEM_WAITING_BIAS (-0x00010000) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - spinlock_t wait_lock; - struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif -}; - -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif -} - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "# beginning down_read\n\t" -LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ - " js 2f\n\t" /* jump if we weren't granted the lock */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " pushl %%edx\n\t" - " call rwsem_down_read_failed\n\t" - " popl %%edx\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END - "# ending down_read\n\t" - : "+m"(sem->count) - : "a"(sem) - : "memory", "cc"); -} - -/* - * lock for writing - */ -static inline void __down_write(struct rw_semaphore *sem) -{ - int tmp; - - tmp = RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - "# beginning down_write\n\t" -LOCK_PREFIX " xadd %0,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ - " testl %0,%0\n\t" /* was the count 0 before? */ - " jnz 2f\n\t" /* jump if we weren't granted the lock */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " call rwsem_down_write_failed\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END - "# ending down_write" - : "+d"(tmp), "+m"(sem->count) - : "a"(sem) - : "memory", "cc"); -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; - __asm__ __volatile__( - "# beginning __up_read\n\t" -LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ - " js 2f\n\t" /* jump if the lock is being waited upon */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* do nothing if still outstanding active readers */ - " jnz 1b\n\t" - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END - "# ending __up_read\n" - : "+m"(sem->count), "+d"(tmp) - : "a"(sem) - : "memory", "cc"); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "# beginning __up_write\n\t" - " movl %2,%%edx\n\t" -LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ - " jnz 2f\n\t" /* jump if the lock is being waited upon */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* did the active count reduce to 0? */ - " jnz 1b\n\t" /* jump back if not */ - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END - "# ending __up_write\n" - : "+m"(sem->count) - : "a"(sem), "i"(-RWSEM_ACTIVE_WRITE_BIAS) - : "memory", "cc", "edx"); -} - -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) -{ - __asm__ __volatile__( -LOCK_PREFIX "addl %1,%0" - :"=m"(sem->count) - :"ir"(delta), "m"(sem->count)); -} - -/* - * implement exchange and add functionality - */ -static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) -{ - int tmp = delta; - - __asm__ __volatile__( -LOCK_PREFIX "xadd %0,(%2)" - : "+r"(tmp), "=m"(sem->count) - : "r"(sem), "m"(sem->count) - : "memory"); - - return tmp+delta; -} - -#endif /* __KERNEL__ */ -#endif /* _I386_RWSEM_H */ diff -urNp 2.4.19rc2/include/asm-ppc/rwsem.h 2.4.19rc2aa1/include/asm-ppc/rwsem.h --- 2.4.19rc2/include/asm-ppc/rwsem.h Tue Jan 22 18:51:12 2002 +++ 2.4.19rc2aa1/include/asm-ppc/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,137 +0,0 @@ -/* - * BK Id: SCCS/s.rwsem.h 1.6 05/17/01 18:14:25 cort - */ -/* - * include/asm-ppc/rwsem.h: R/W semaphores for PPC using the stuff - * in lib/rwsem.c. Adapted largely from include/asm-i386/rwsem.h - * by Paul Mackerras . - */ - -#ifndef _PPC_RWSEM_H -#define _PPC_RWSEM_H - -#ifdef __KERNEL__ -#include -#include -#include -#include - -/* - * the semaphore definition - */ -struct rw_semaphore { - /* XXX this should be able to be an atomic_t -- paulus */ - signed long count; -#define RWSEM_UNLOCKED_VALUE 0x00000000 -#define RWSEM_ACTIVE_BIAS 0x00000001 -#define RWSEM_ACTIVE_MASK 0x0000ffff -#define RWSEM_WAITING_BIAS (-0x00010000) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - spinlock_t wait_lock; - struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif -}; - -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - -#define __RWSEM_INITIALIZER(name) \ - { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, \ - LIST_HEAD_INIT((name).wait_list) \ - __RWSEM_DEBUG_INIT } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem); - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif -} - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - if (atomic_inc_return((atomic_t *)(&sem->count)) >= 0) - smp_wmb(); - else - rwsem_down_read_failed(sem); -} - -/* - * lock for writing - */ -static inline void __down_write(struct rw_semaphore *sem) -{ - int tmp; - - tmp = atomic_add_return(RWSEM_ACTIVE_WRITE_BIAS, - (atomic_t *)(&sem->count)); - if (tmp == RWSEM_ACTIVE_WRITE_BIAS) - smp_wmb(); - else - rwsem_down_write_failed(sem); -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - int tmp; - - smp_wmb(); - tmp = atomic_dec_return((atomic_t *)(&sem->count)); - if (tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0) - rwsem_wake(sem); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - smp_wmb(); - if (atomic_sub_return(RWSEM_ACTIVE_WRITE_BIAS, - (atomic_t *)(&sem->count)) < 0) - rwsem_wake(sem); -} - -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) -{ - atomic_add(delta, (atomic_t *)(&sem->count)); -} - -/* - * implement exchange and add functionality - */ -static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) -{ - smp_mb(); - return atomic_add_return(delta, (atomic_t *)(&sem->count)); -} - -#endif /* __KERNEL__ */ -#endif /* _PPC_RWSEM_XADD_H */ diff -urNp 2.4.19rc2/include/asm-s390/rwsem.h 2.4.19rc2aa1/include/asm-s390/rwsem.h --- 2.4.19rc2/include/asm-s390/rwsem.h Wed Jul 17 13:28:50 2002 +++ 2.4.19rc2aa1/include/asm-s390/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,209 +0,0 @@ -#ifndef _S390_RWSEM_H -#define _S390_RWSEM_H - -/* - * include/asm-s390/rwsem.h - * - * S390 version - * Copyright (C) 2002 IBM Deutschland Entwicklung GmbH, IBM Corporation - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) - * - * Based on asm-alpha/semaphore.h and asm-i386/rwsem.h - */ - -/* - * - * The MSW of the count is the negated number of active writers and waiting - * lockers, and the LSW is the total number of active locks - * - * The lock count is initialized to 0 (no active and no waiting lockers). - * - * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an - * uncontended lock. This can be determined because XADD returns the old value. - * Readers increment by 1 and see a positive value when uncontended, negative - * if there are writers (and maybe) readers waiting (in which case it goes to - * sleep). - * - * The value of WAITING_BIAS supports up to 32766 waiting processes. This can - * be extended to 65534 by manually checking the whole MSW rather than relying - * on the S flag. - * - * The value of ACTIVE_BIAS supports up to 65535 active processes. - * - * This should be totally fair - if anything is waiting, a process that wants a - * lock will go to the back of the queue. When the currently active lock is - * released, if there's a writer at the front of the queue, then that and only - * that will be woken up; if there's a bunch of consequtive readers at the - * front, then they'll all be woken up, but no other readers will be. - */ - -#ifndef _LINUX_RWSEM_H -#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead -#endif - -#ifdef __KERNEL__ - -#include -#include - -struct rwsem_waiter; - -extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); - -/* - * the semaphore definition - */ -struct rw_semaphore { - signed long count; - spinlock_t wait_lock; - struct list_head wait_list; -}; - -#define RWSEM_UNLOCKED_VALUE 0x00000000 -#define RWSEM_ACTIVE_BIAS 0x00000001 -#define RWSEM_ACTIVE_MASK 0x0000ffff -#define RWSEM_WAITING_BIAS (-0x00010000) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - -/* - * initialisation - */ -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -} - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " ahi %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "i" (RWSEM_ACTIVE_READ_BIAS) - : "cc", "memory" ); - if (old < 0) - rwsem_down_read_failed(sem); -} - -/* - * lock for writing - */ -static inline void __down_write(struct rw_semaphore *sem) -{ - signed long old, new, tmp; - - tmp = RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " a %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "m" (tmp) - : "cc", "memory" ); - if (old != 0) - rwsem_down_write_failed(sem); -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " ahi %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "i" (-RWSEM_ACTIVE_READ_BIAS) - : "cc", "memory" ); - if (new < 0) - if ((new & RWSEM_ACTIVE_MASK) == 0) - rwsem_wake(sem); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - signed long old, new, tmp; - - tmp = -RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " a %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "m" (tmp) - : "cc", "memory" ); - if (new < 0) - if ((new & RWSEM_ACTIVE_MASK) == 0) - rwsem_wake(sem); -} - -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " ar %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "d" (delta) - : "cc", "memory" ); -} - -/* - * implement exchange and add functionality - */ -static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " l %0,0(%2)\n" - "0: lr %1,%0\n" - " ar %1,%3\n" - " cs %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "d" (delta) - : "cc", "memory" ); - return new; -} - -#endif /* __KERNEL__ */ -#endif /* _S390_RWSEM_H */ diff -urNp 2.4.19rc2/include/asm-s390x/rwsem.h 2.4.19rc2aa1/include/asm-s390x/rwsem.h --- 2.4.19rc2/include/asm-s390x/rwsem.h Wed Jul 17 13:28:50 2002 +++ 2.4.19rc2aa1/include/asm-s390x/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,209 +0,0 @@ -#ifndef _S390X_RWSEM_H -#define _S390X_RWSEM_H - -/* - * include/asm-s390x/rwsem.h - * - * S390 version - * Copyright (C) 2002 IBM Deutschland Entwicklung GmbH, IBM Corporation - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com) - * - * Based on asm-alpha/semaphore.h and asm-i386/rwsem.h - */ - -/* - * - * The MSW of the count is the negated number of active writers and waiting - * lockers, and the LSW is the total number of active locks - * - * The lock count is initialized to 0 (no active and no waiting lockers). - * - * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an - * uncontended lock. This can be determined because XADD returns the old value. - * Readers increment by 1 and see a positive value when uncontended, negative - * if there are writers (and maybe) readers waiting (in which case it goes to - * sleep). - * - * The value of WAITING_BIAS supports up to 32766 waiting processes. This can - * be extended to 65534 by manually checking the whole MSW rather than relying - * on the S flag. - * - * The value of ACTIVE_BIAS supports up to 65535 active processes. - * - * This should be totally fair - if anything is waiting, a process that wants a - * lock will go to the back of the queue. When the currently active lock is - * released, if there's a writer at the front of the queue, then that and only - * that will be woken up; if there's a bunch of consequtive readers at the - * front, then they'll all be woken up, but no other readers will be. - */ - -#ifndef _LINUX_RWSEM_H -#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead -#endif - -#ifdef __KERNEL__ - -#include -#include - -struct rwsem_waiter; - -extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); - -/* - * the semaphore definition - */ -struct rw_semaphore { - signed long count; - spinlock_t wait_lock; - struct list_head wait_list; -}; - -#define RWSEM_UNLOCKED_VALUE 0x0000000000000000L -#define RWSEM_ACTIVE_BIAS 0x0000000000000001L -#define RWSEM_ACTIVE_MASK 0x00000000ffffffffL -#define RWSEM_WAITING_BIAS (-0x0000000100000000L) -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - -/* - * initialisation - */ -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -} - -/* - * lock for reading - */ -static inline void __down_read(struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " aghi %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "i" (RWSEM_ACTIVE_READ_BIAS) - : "cc", "memory" ); - if (old < 0) - rwsem_down_read_failed(sem); -} - -/* - * lock for writing - */ -static inline void __down_write(struct rw_semaphore *sem) -{ - signed long old, new, tmp; - - tmp = RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " ag %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "m" (tmp) - : "cc", "memory" ); - if (old != 0) - rwsem_down_write_failed(sem); -} - -/* - * unlock after reading - */ -static inline void __up_read(struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " aghi %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "i" (-RWSEM_ACTIVE_READ_BIAS) - : "cc", "memory" ); - if (new < 0) - if ((int) new == 0) - rwsem_wake(sem); -} - -/* - * unlock after writing - */ -static inline void __up_write(struct rw_semaphore *sem) -{ - signed long old, new, tmp; - - tmp = -RWSEM_ACTIVE_WRITE_BIAS; - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " ag %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "m" (tmp) - : "cc", "memory" ); - if (new < 0) - if ((int) new == 0) - rwsem_wake(sem); -} - -/* - * implement atomic add functionality - */ -static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " agr %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "d" (delta) - : "cc", "memory" ); -} - -/* - * implement exchange and add functionality - */ -static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) -{ - signed long old, new; - - __asm__ __volatile__( - " lg %0,0(%2)\n" - "0: lgr %1,%0\n" - " agr %1,%3\n" - " csg %0,%1,0(%2)\n" - " jl 0b" - : "=&d" (old), "=&d" (new) - : "a" (&sem->count), "d" (delta) - : "cc", "memory" ); - return new; -} - -#endif /* __KERNEL__ */ -#endif /* _S390X_RWSEM_H */ diff -urNp 2.4.19rc2/include/asm-sparc64/rwsem.h 2.4.19rc2aa1/include/asm-sparc64/rwsem.h --- 2.4.19rc2/include/asm-sparc64/rwsem.h Tue Jan 22 18:56:29 2002 +++ 2.4.19rc2aa1/include/asm-sparc64/rwsem.h Thu Jan 1 01:00:00 1970 @@ -1,233 +0,0 @@ -/* $Id: rwsem.h,v 1.5 2001/11/18 00:12:56 davem Exp $ - * rwsem.h: R/W semaphores implemented using CAS - * - * Written by David S. Miller (davem@redhat.com), 2001. - * Derived from asm-i386/rwsem.h - */ -#ifndef _SPARC64_RWSEM_H -#define _SPARC64_RWSEM_H - -#ifndef _LINUX_RWSEM_H -#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead -#endif - -#ifdef __KERNEL__ - -#include -#include - -struct rwsem_waiter; - -extern struct rw_semaphore *FASTCALL(rwsem_down_read_failed(struct rw_semaphore *sem)); -extern struct rw_semaphore *FASTCALL(rwsem_down_write_failed(struct rw_semaphore *sem)); -extern struct rw_semaphore *FASTCALL(rwsem_wake(struct rw_semaphore *)); - -struct rw_semaphore { - signed int count; -#define RWSEM_UNLOCKED_VALUE 0x00000000 -#define RWSEM_ACTIVE_BIAS 0x00000001 -#define RWSEM_ACTIVE_MASK 0x0000ffff -#define RWSEM_WAITING_BIAS 0xffff0000 -#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS -#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - spinlock_t wait_lock; - struct list_head wait_list; -}; - -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -static inline void init_rwsem(struct rw_semaphore *sem) -{ - sem->count = RWSEM_UNLOCKED_VALUE; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -} - -static inline void __down_read(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "! beginning __down_read\n" - "1:\tlduw [%0], %%g5\n\t" - "add %%g5, 1, %%g7\n\t" - "cas [%0], %%g5, %%g7\n\t" - "cmp %%g5, %%g7\n\t" - "bne,pn %%icc, 1b\n\t" - " add %%g7, 1, %%g7\n\t" - "cmp %%g7, 0\n\t" - "bl,pn %%icc, 3f\n\t" - " membar #StoreLoad | #StoreStore\n" - "2:\n\t" - ".subsection 2\n" - "3:\tmov %0, %%g5\n\t" - "save %%sp, -160, %%sp\n\t" - "mov %%g1, %%l1\n\t" - "mov %%g2, %%l2\n\t" - "mov %%g3, %%l3\n\t" - "call %1\n\t" - " mov %%g5, %%o0\n\t" - "mov %%l1, %%g1\n\t" - "mov %%l2, %%g2\n\t" - "ba,pt %%xcc, 2b\n\t" - " restore %%l3, %%g0, %%g3\n\t" - ".previous\n\t" - "! ending __down_read" - : : "r" (sem), "i" (rwsem_down_read_failed) - : "g5", "g7", "memory", "cc"); -} - -static inline void __down_write(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "! beginning __down_write\n\t" - "sethi %%hi(%2), %%g1\n\t" - "or %%g1, %%lo(%2), %%g1\n" - "1:\tlduw [%0], %%g5\n\t" - "add %%g5, %%g1, %%g7\n\t" - "cas [%0], %%g5, %%g7\n\t" - "cmp %%g5, %%g7\n\t" - "bne,pn %%icc, 1b\n\t" - " cmp %%g7, 0\n\t" - "bne,pn %%icc, 3f\n\t" - " membar #StoreLoad | #StoreStore\n" - "2:\n\t" - ".subsection 2\n" - "3:\tmov %0, %%g5\n\t" - "save %%sp, -160, %%sp\n\t" - "mov %%g2, %%l2\n\t" - "mov %%g3, %%l3\n\t" - "call %1\n\t" - " mov %%g5, %%o0\n\t" - "mov %%l2, %%g2\n\t" - "ba,pt %%xcc, 2b\n\t" - " restore %%l3, %%g0, %%g3\n\t" - ".previous\n\t" - "! ending __down_write" - : : "r" (sem), "i" (rwsem_down_write_failed), - "i" (RWSEM_ACTIVE_WRITE_BIAS) - : "g1", "g5", "g7", "memory", "cc"); -} - -static inline void __up_read(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "! beginning __up_read\n\t" - "1:\tlduw [%0], %%g5\n\t" - "sub %%g5, 1, %%g7\n\t" - "cas [%0], %%g5, %%g7\n\t" - "cmp %%g5, %%g7\n\t" - "bne,pn %%icc, 1b\n\t" - " cmp %%g7, 0\n\t" - "bl,pn %%icc, 3f\n\t" - " membar #StoreLoad | #StoreStore\n" - "2:\n\t" - ".subsection 2\n" - "3:\tsethi %%hi(%2), %%g1\n\t" - "sub %%g7, 1, %%g7\n\t" - "or %%g1, %%lo(%2), %%g1\n\t" - "andcc %%g7, %%g1, %%g0\n\t" - "bne,pn %%icc, 2b\n\t" - " mov %0, %%g5\n\t" - "save %%sp, -160, %%sp\n\t" - "mov %%g2, %%l2\n\t" - "mov %%g3, %%l3\n\t" - "call %1\n\t" - " mov %%g5, %%o0\n\t" - "mov %%l2, %%g2\n\t" - "ba,pt %%xcc, 2b\n\t" - " restore %%l3, %%g0, %%g3\n\t" - ".previous\n\t" - "! ending __up_read" - : : "r" (sem), "i" (rwsem_wake), - "i" (RWSEM_ACTIVE_MASK) - : "g1", "g5", "g7", "memory", "cc"); -} - -static inline void __up_write(struct rw_semaphore *sem) -{ - __asm__ __volatile__( - "! beginning __up_write\n\t" - "sethi %%hi(%2), %%g1\n\t" - "or %%g1, %%lo(%2), %%g1\n" - "1:\tlduw [%0], %%g5\n\t" - "sub %%g5, %%g1, %%g7\n\t" - "cas [%0], %%g5, %%g7\n\t" - "cmp %%g5, %%g7\n\t" - "bne,pn %%icc, 1b\n\t" - " sub %%g7, %%g1, %%g7\n\t" - "cmp %%g7, 0\n\t" - "bl,pn %%icc, 3f\n\t" - " membar #StoreLoad | #StoreStore\n" - "2:\n\t" - ".subsection 2\n" - "3:\tmov %0, %%g5\n\t" - "save %%sp, -160, %%sp\n\t" - "mov %%g2, %%l2\n\t" - "mov %%g3, %%l3\n\t" - "call %1\n\t" - " mov %%g5, %%o0\n\t" - "mov %%l2, %%g2\n\t" - "ba,pt %%xcc, 2b\n\t" - " restore %%l3, %%g0, %%g3\n\t" - ".previous\n\t" - "! ending __up_write" - : : "r" (sem), "i" (rwsem_wake), - "i" (RWSEM_ACTIVE_WRITE_BIAS) - : "g1", "g5", "g7", "memory", "cc"); -} - -static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) -{ - int tmp = delta; - - __asm__ __volatile__( - "1:\tlduw [%2], %%g5\n\t" - "add %%g5, %1, %%g7\n\t" - "cas [%2], %%g5, %%g7\n\t" - "cmp %%g5, %%g7\n\t" - "bne,pn %%icc, 1b\n\t" - " membar #StoreLoad | #StoreStore\n\t" - "mov %%g7, %0\n\t" - : "=&r" (tmp) - : "0" (tmp), "r" (sem) - : "g5", "g7", "memory"); - - return tmp + delta; -} - -#define rwsem_atomic_add rwsem_atomic_update - -static inline __u16 rwsem_cmpxchgw(struct rw_semaphore *sem, __u16 __old, __u16 __new) -{ - u32 old = (sem->count & 0xffff0000) | (u32) __old; - u32 new = (old & 0xffff0000) | (u32) __new; - u32 prev; - -again: - __asm__ __volatile__("cas [%2], %3, %0\n\t" - "membar #StoreLoad | #StoreStore" - : "=&r" (prev) - : "0" (new), "r" (sem), "r" (old) - : "memory"); - - /* To give the same semantics as x86 cmpxchgw, keep trying - * if only the upper 16-bits changed. - */ - if (prev != old && - ((prev & 0xffff) == (old & 0xffff))) - goto again; - - return prev & 0xffff; -} - -static inline signed long rwsem_cmpxchg(struct rw_semaphore *sem, signed long old, signed long new) -{ - return cmpxchg(&sem->count,old,new); -} - -#endif /* __KERNEL__ */ - -#endif /* _SPARC64_RWSEM_H */ diff -urNp 2.4.19rc2/include/linux/rwsem-spinlock.h 2.4.19rc2aa1/include/linux/rwsem-spinlock.h --- 2.4.19rc2/include/linux/rwsem-spinlock.h Tue Jul 16 23:56:42 2002 +++ 2.4.19rc2aa1/include/linux/rwsem-spinlock.h Thu Jan 1 01:00:00 1970 @@ -1,62 +0,0 @@ -/* rwsem-spinlock.h: fallback C implementation - * - * Copyright (c) 2001 David Howells (dhowells@redhat.com). - * - Derived partially from ideas by Andrea Arcangeli - * - Derived also from comments by Linus - */ - -#ifndef _LINUX_RWSEM_SPINLOCK_H -#define _LINUX_RWSEM_SPINLOCK_H - -#ifndef _LINUX_RWSEM_H -#error please dont include linux/rwsem-spinlock.h directly, use linux/rwsem.h instead -#endif - -#include -#include - -#ifdef __KERNEL__ - -#include - -struct rwsem_waiter; - -/* - * the rw-semaphore definition - * - if activity is 0 then there are no active readers or writers - * - if activity is +ve then that is the number of active readers - * - if activity is -1 then there is one active writer - * - if wait_list is not empty, then there are processes waiting for the semaphore - */ -struct rw_semaphore { - __s32 activity; - spinlock_t wait_lock; - struct list_head wait_list; -#if RWSEM_DEBUG - int debug; -#endif -}; - -/* - * initialisation - */ -#if RWSEM_DEBUG -#define __RWSEM_DEBUG_INIT , 0 -#else -#define __RWSEM_DEBUG_INIT /* */ -#endif - -#define __RWSEM_INITIALIZER(name) \ -{ 0, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) __RWSEM_DEBUG_INIT } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -extern void FASTCALL(init_rwsem(struct rw_semaphore *sem)); -extern void FASTCALL(__down_read(struct rw_semaphore *sem)); -extern void FASTCALL(__down_write(struct rw_semaphore *sem)); -extern void FASTCALL(__up_read(struct rw_semaphore *sem)); -extern void FASTCALL(__up_write(struct rw_semaphore *sem)); - -#endif /* __KERNEL__ */ -#endif /* _LINUX_RWSEM_SPINLOCK_H */ diff -urNp 2.4.19rc2/include/linux/rwsem.h 2.4.19rc2aa1/include/linux/rwsem.h --- 2.4.19rc2/include/linux/rwsem.h Tue Jul 16 23:56:42 2002 +++ 2.4.19rc2aa1/include/linux/rwsem.h Wed Jul 17 14:28:57 2002 @@ -1,80 +1,120 @@ -/* rwsem.h: R/W semaphores, public interface - * - * Written by David Howells (dhowells@redhat.com). - * Derived from asm-i386/semaphore.h - */ - #ifndef _LINUX_RWSEM_H #define _LINUX_RWSEM_H -#include - -#define RWSEM_DEBUG 0 - #ifdef __KERNEL__ -#include -#include +#include #include -#include -#include -struct rw_semaphore; - -#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK -#include /* use a generic implementation */ -#else -#include /* use an arch-specific implementation */ +struct rw_semaphore +{ + spinlock_t lock; + long count; +#define RWSEM_READ_BIAS 1 +#define RWSEM_WRITE_BIAS (~(~0UL >> (BITS_PER_LONG>>1))) + struct list_head wait; +#if RWSEM_DEBUG + long __magic; #endif +}; -#ifndef rwsemtrace #if RWSEM_DEBUG -extern void FASTCALL(rwsemtrace(struct rw_semaphore *sem, const char *str)); +#define __SEM_DEBUG_INIT(name) \ + , (long)&(name).__magic +#define RWSEM_MAGIC(x) \ + do { \ + if ((x) != (long)&(x)) { \ + printk("rwsem bad magic %lx (should be %lx), ", \ + (long)x, (long)&(x)); \ + BUG(); \ + } \ + } while (0) #else -#define rwsemtrace(SEM,FMT) +#define __SEM_DEBUG_INIT(name) +#define CHECK_MAGIC(x) #endif + +#define __RWSEM_INITIALIZER(name, count) \ +{ \ + SPIN_LOCK_UNLOCKED, \ + (count), \ + LIST_HEAD_INIT((name).wait) \ + __SEM_DEBUG_INIT(name) \ +} +#define RWSEM_INITIALIZER(name) __RWSEM_INITIALIZER(name, 0) + +#define __DECLARE_RWSEM(name, count) \ + struct rw_semaphore name = __RWSEM_INITIALIZER(name, count) +#define DECLARE_RWSEM(name) __DECLARE_RWSEM(name, 0) +#define DECLARE_RWSEM_READ_LOCKED(name) __DECLARE_RWSEM(name, RWSEM_READ_BIAS) +#define DECLARE_RWSEM_WRITE_LOCKED(name) __DECLARE_RWSEM(name, RWSEM_WRITE_BIAS) + +#define RWSEM_READ_BLOCKING_BIAS (RWSEM_WRITE_BIAS-RWSEM_READ_BIAS) +#define RWSEM_WRITE_BLOCKING_BIAS (0) + +#define RWSEM_READ_MASK (~RWSEM_WRITE_BIAS) +#define RWSEM_WRITE_MASK (RWSEM_WRITE_BIAS) + +extern void FASTCALL(rwsem_down_failed(struct rw_semaphore *, long)); +extern void FASTCALL(rwsem_wake(struct rw_semaphore *)); + +static inline void init_rwsem(struct rw_semaphore *sem) +{ + spin_lock_init(&sem->lock); + sem->count = 0; + INIT_LIST_HEAD(&sem->wait); +#if RWSEM_DEBUG + sem->__magic = (long)&sem->__magic; #endif +} -/* - * lock for reading - */ static inline void down_read(struct rw_semaphore *sem) { - rwsemtrace(sem,"Entering down_read"); - __down_read(sem); - rwsemtrace(sem,"Leaving down_read"); + long count; + CHECK_MAGIC(sem->__magic); + + spin_lock(&sem->lock); + count = sem->count; + sem->count += RWSEM_READ_BIAS; + if (unlikely(count < 0)) + rwsem_down_failed(sem, RWSEM_READ_BLOCKING_BIAS); + spin_unlock(&sem->lock); } -/* - * lock for writing - */ static inline void down_write(struct rw_semaphore *sem) { - rwsemtrace(sem,"Entering down_write"); - __down_write(sem); - rwsemtrace(sem,"Leaving down_write"); + long count; + CHECK_MAGIC(sem->__magic); + + spin_lock(&sem->lock); + count = sem->count; + sem->count += RWSEM_WRITE_BIAS; + if (unlikely(count)) + rwsem_down_failed(sem, RWSEM_WRITE_BLOCKING_BIAS); + spin_unlock(&sem->lock); } -/* - * release a read lock - */ static inline void up_read(struct rw_semaphore *sem) { - rwsemtrace(sem,"Entering up_read"); - __up_read(sem); - rwsemtrace(sem,"Leaving up_read"); + CHECK_MAGIC(sem->__magic); + + spin_lock(&sem->lock); + sem->count -= RWSEM_READ_BIAS; + if (unlikely(sem->count < 0 && !(sem->count & RWSEM_READ_MASK))) + rwsem_wake(sem); + spin_unlock(&sem->lock); } -/* - * release a write lock - */ static inline void up_write(struct rw_semaphore *sem) { - rwsemtrace(sem,"Entering up_write"); - __up_write(sem); - rwsemtrace(sem,"Leaving up_write"); -} + CHECK_MAGIC(sem->__magic); + spin_lock(&sem->lock); + sem->count -= RWSEM_WRITE_BIAS; + if (unlikely(sem->count)) + rwsem_wake(sem); + spin_unlock(&sem->lock); +} #endif /* __KERNEL__ */ #endif /* _LINUX_RWSEM_H */ diff -urNp 2.4.19rc2/include/linux/sched.h 2.4.19rc2aa1/include/linux/sched.h --- 2.4.19rc2/include/linux/sched.h Wed Jul 17 13:28:51 2002 +++ 2.4.19rc2aa1/include/linux/sched.h Wed Jul 17 14:28:57 2002 @@ -245,7 +245,7 @@ extern int mmlist_nr; pgd: swapper_pg_dir, \ mm_users: ATOMIC_INIT(2), \ mm_count: ATOMIC_INIT(1), \ - mmap_sem: __RWSEM_INITIALIZER(name.mmap_sem), \ + mmap_sem: RWSEM_INITIALIZER(name.mmap_sem), \ page_table_lock: SPIN_LOCK_UNLOCKED, \ mmlist: LIST_HEAD_INIT(name.mmlist), \ } diff -urNp 2.4.19rc2/lib/Makefile 2.4.19rc2aa1/lib/Makefile --- 2.4.19rc2/lib/Makefile Wed Jul 17 13:28:51 2002 +++ 2.4.19rc2aa1/lib/Makefile Wed Jul 17 14:28:57 2002 @@ -8,12 +8,9 @@ L_TARGET := lib.a -export-objs := cmdline.o dec_and_lock.o rwsem-spinlock.o rwsem.o rbtree.o +export-objs := cmdline.o dec_and_lock.o rwsem.o rbtree.o -obj-y := errno.o ctype.o string.o vsprintf.o brlock.o cmdline.o bust_spinlocks.o rbtree.o - -obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o -obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o +obj-y := errno.o ctype.o string.o vsprintf.o brlock.o cmdline.o bust_spinlocks.o rbtree.o rwsem.o ifneq ($(CONFIG_HAVE_DEC_LOCK),y) obj-y += dec_and_lock.o diff -urNp 2.4.19rc2/lib/rwsem-spinlock.c 2.4.19rc2aa1/lib/rwsem-spinlock.c --- 2.4.19rc2/lib/rwsem-spinlock.c Tue May 1 19:35:33 2001 +++ 2.4.19rc2aa1/lib/rwsem-spinlock.c Thu Jan 1 01:00:00 1970 @@ -1,239 +0,0 @@ -/* rwsem-spinlock.c: R/W semaphores: contention handling functions for generic spinlock - * implementation - * - * Copyright (c) 2001 David Howells (dhowells@redhat.com). - * - Derived partially from idea by Andrea Arcangeli - * - Derived also from comments by Linus - */ -#include -#include -#include - -struct rwsem_waiter { - struct list_head list; - struct task_struct *task; - unsigned int flags; -#define RWSEM_WAITING_FOR_READ 0x00000001 -#define RWSEM_WAITING_FOR_WRITE 0x00000002 -}; - -#if RWSEM_DEBUG -void rwsemtrace(struct rw_semaphore *sem, const char *str) -{ - if (sem->debug) - printk("[%d] %s({%d,%d})\n", - current->pid,str,sem->activity,list_empty(&sem->wait_list)?0:1); -} -#endif - -/* - * initialise the semaphore - */ -void init_rwsem(struct rw_semaphore *sem) -{ - sem->activity = 0; - spin_lock_init(&sem->wait_lock); - INIT_LIST_HEAD(&sem->wait_list); -#if RWSEM_DEBUG - sem->debug = 0; -#endif -} - -/* - * handle the lock being released whilst there are processes blocked on it that can now run - * - if we come here, then: - * - the 'active count' _reached_ zero - * - the 'waiting count' is non-zero - * - the spinlock must be held by the caller - * - woken process blocks are discarded from the list after having flags zeroised - */ -static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem) -{ - struct rwsem_waiter *waiter; - int woken; - - rwsemtrace(sem,"Entering __rwsem_do_wake"); - - waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); - - /* try to grant a single write lock if there's a writer at the front of the queue - * - we leave the 'waiting count' incremented to signify potential contention - */ - if (waiter->flags & RWSEM_WAITING_FOR_WRITE) { - sem->activity = -1; - list_del(&waiter->list); - waiter->flags = 0; - wake_up_process(waiter->task); - goto out; - } - - /* grant an infinite number of read locks to the readers at the front of the queue */ - woken = 0; - do { - list_del(&waiter->list); - waiter->flags = 0; - wake_up_process(waiter->task); - woken++; - if (list_empty(&sem->wait_list)) - break; - waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); - } while (waiter->flags&RWSEM_WAITING_FOR_READ); - - sem->activity += woken; - - out: - rwsemtrace(sem,"Leaving __rwsem_do_wake"); - return sem; -} - -/* - * wake a single writer - */ -static inline struct rw_semaphore *__rwsem_wake_one_writer(struct rw_semaphore *sem) -{ - struct rwsem_waiter *waiter; - - sem->activity = -1; - - waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); - list_del(&waiter->list); - - waiter->flags = 0; - wake_up_process(waiter->task); - return sem; -} - -/* - * get a read lock on the semaphore - */ -void __down_read(struct rw_semaphore *sem) -{ - struct rwsem_waiter waiter; - struct task_struct *tsk; - - rwsemtrace(sem,"Entering __down_read"); - - spin_lock(&sem->wait_lock); - - if (sem->activity>=0 && list_empty(&sem->wait_list)) { - /* granted */ - sem->activity++; - spin_unlock(&sem->wait_lock); - goto out; - } - - tsk = current; - set_task_state(tsk,TASK_UNINTERRUPTIBLE); - - /* set up my own style of waitqueue */ - waiter.task = tsk; - waiter.flags = RWSEM_WAITING_FOR_READ; - - list_add_tail(&waiter.list,&sem->wait_list); - - /* we don't need to touch the semaphore struct anymore */ - spin_unlock(&sem->wait_lock); - - /* wait to be given the lock */ - for (;;) { - if (!waiter.flags) - break; - schedule(); - set_task_state(tsk, TASK_UNINTERRUPTIBLE); - } - - tsk->state = TASK_RUNNING; - - out: - rwsemtrace(sem,"Leaving __down_read"); -} - -/* - * get a write lock on the semaphore - * - note that we increment the waiting count anyway to indicate an exclusive lock - */ -void __down_write(struct rw_semaphore *sem) -{ - struct rwsem_waiter waiter; - struct task_struct *tsk; - - rwsemtrace(sem,"Entering __down_write"); - - spin_lock(&sem->wait_lock); - - if (sem->activity==0 && list_empty(&sem->wait_list)) { - /* granted */ - sem->activity = -1; - spin_unlock(&sem->wait_lock); - goto out; - } - - tsk = current; - set_task_state(tsk,TASK_UNINTERRUPTIBLE); - - /* set up my own style of waitqueue */ - waiter.task = tsk; - waiter.flags = RWSEM_WAITING_FOR_WRITE; - - list_add_tail(&waiter.list,&sem->wait_list); - - /* we don't need to touch the semaphore struct anymore */ - spin_unlock(&sem->wait_lock); - - /* wait to be given the lock */ - for (;;) { - if (!waiter.flags) - break; - schedule(); - set_task_state(tsk, TASK_UNINTERRUPTIBLE); - } - - tsk->state = TASK_RUNNING; - - out: - rwsemtrace(sem,"Leaving __down_write"); -} - -/* - * release a read lock on the semaphore - */ -void __up_read(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering __up_read"); - - spin_lock(&sem->wait_lock); - - if (--sem->activity==0 && !list_empty(&sem->wait_list)) - sem = __rwsem_wake_one_writer(sem); - - spin_unlock(&sem->wait_lock); - - rwsemtrace(sem,"Leaving __up_read"); -} - -/* - * release a write lock on the semaphore - */ -void __up_write(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering __up_write"); - - spin_lock(&sem->wait_lock); - - sem->activity = 0; - if (!list_empty(&sem->wait_list)) - sem = __rwsem_do_wake(sem); - - spin_unlock(&sem->wait_lock); - - rwsemtrace(sem,"Leaving __up_write"); -} - -EXPORT_SYMBOL(init_rwsem); -EXPORT_SYMBOL(__down_read); -EXPORT_SYMBOL(__down_write); -EXPORT_SYMBOL(__up_read); -EXPORT_SYMBOL(__up_write); -#if RWSEM_DEBUG -EXPORT_SYMBOL(rwsemtrace); -#endif diff -urNp 2.4.19rc2/lib/rwsem.c 2.4.19rc2aa1/lib/rwsem.c --- 2.4.19rc2/lib/rwsem.c Tue Jan 22 18:52:17 2002 +++ 2.4.19rc2aa1/lib/rwsem.c Wed Jul 17 14:28:57 2002 @@ -1,210 +1,63 @@ -/* rwsem.c: R/W semaphores: contention handling functions - * - * Written by David Howells (dhowells@redhat.com). - * Derived from arch/i386/kernel/semaphore.c +/* + * rw_semaphores generic spinlock version + * Copyright (C) 2001 Andrea Arcangeli SuSE */ -#include + #include #include +#include -struct rwsem_waiter { - struct list_head list; - struct task_struct *task; - unsigned int flags; -#define RWSEM_WAITING_FOR_READ 0x00000001 -#define RWSEM_WAITING_FOR_WRITE 0x00000002 +struct rwsem_wait_queue { + unsigned long retire; + struct task_struct * task; + struct list_head task_list; }; -#if RWSEM_DEBUG -#undef rwsemtrace -void rwsemtrace(struct rw_semaphore *sem, const char *str) -{ - printk("sem=%p\n",sem); - printk("(sem)=%08lx\n",sem->count); - if (sem->debug) - printk("[%d] %s({%08lx})\n",current->pid,str,sem->count); -} -#endif - -/* - * handle the lock being released whilst there are processes blocked on it that can now run - * - if we come here, then: - * - the 'active part' of the count (&0x0000ffff) reached zero but has been re-incremented - * - the 'waiting part' of the count (&0xffff0000) is negative (and will still be so) - * - there must be someone on the queue - * - the spinlock must be held by the caller - * - woken process blocks are discarded from the list after having flags zeroised - */ -static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem) -{ - struct rwsem_waiter *waiter; - struct list_head *next; - signed long oldcount; - int woken, loop; - - rwsemtrace(sem,"Entering __rwsem_do_wake"); - - /* only wake someone up if we can transition the active part of the count from 0 -> 1 */ - try_again: - oldcount = rwsem_atomic_update(RWSEM_ACTIVE_BIAS,sem) - RWSEM_ACTIVE_BIAS; - if (oldcount & RWSEM_ACTIVE_MASK) - goto undo; - - waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list); - - /* try to grant a single write lock if there's a writer at the front of the queue - * - note we leave the 'active part' of the count incremented by 1 and the waiting part - * incremented by 0x00010000 - */ - if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE)) - goto readers_only; - - list_del(&waiter->list); - waiter->flags = 0; - wake_up_process(waiter->task); - goto out; - - /* grant an infinite number of read locks to the readers at the front of the queue - * - note we increment the 'active part' of the count by the number of readers (less one - * for the activity decrement we've already done) before waking any processes up - */ - readers_only: - woken = 0; - do { - woken++; - - if (waiter->list.next==&sem->wait_list) - break; - - waiter = list_entry(waiter->list.next,struct rwsem_waiter,list); - - } while (waiter->flags & RWSEM_WAITING_FOR_READ); - - loop = woken; - woken *= RWSEM_ACTIVE_BIAS-RWSEM_WAITING_BIAS; - woken -= RWSEM_ACTIVE_BIAS; - rwsem_atomic_add(woken,sem); - - next = sem->wait_list.next; - for (; loop>0; loop--) { - waiter = list_entry(next,struct rwsem_waiter,list); - next = waiter->list.next; - waiter->flags = 0; - wake_up_process(waiter->task); - } - - sem->wait_list.next = next; - next->prev = &sem->wait_list; - - out: - rwsemtrace(sem,"Leaving __rwsem_do_wake"); - return sem; - - /* undo the change to count, but check for a transition 1->0 */ - undo: - if (rwsem_atomic_update(-RWSEM_ACTIVE_BIAS,sem)!=0) - goto out; - goto try_again; -} - -/* - * wait for a lock to be granted - */ -static inline struct rw_semaphore *rwsem_down_failed_common(struct rw_semaphore *sem, - struct rwsem_waiter *waiter, - signed long adjustment) +void rwsem_down_failed(struct rw_semaphore *sem, long retire) { struct task_struct *tsk = current; - signed long count; - - set_task_state(tsk,TASK_UNINTERRUPTIBLE); - - /* set up my own style of waitqueue */ - spin_lock(&sem->wait_lock); - waiter->task = tsk; - - list_add_tail(&waiter->list,&sem->wait_list); - - /* note that we're now waiting on the lock, but no longer actively read-locking */ - count = rwsem_atomic_update(adjustment,sem); - - /* if there are no longer active locks, wake the front queued process(es) up - * - it might even be this process, since the waker takes a more active part - */ - if (!(count & RWSEM_ACTIVE_MASK)) - sem = __rwsem_do_wake(sem); + struct rwsem_wait_queue wait; - spin_unlock(&sem->wait_lock); + sem->count += retire; + wait.retire = retire; + wait.task = tsk; + INIT_LIST_HEAD(&wait.task_list); + list_add(&wait.task_list, &sem->wait); - /* wait to be given the lock */ - for (;;) { - if (!waiter->flags) - break; + do { + __set_task_state(tsk, TASK_UNINTERRUPTIBLE); + spin_unlock(&sem->lock); schedule(); - set_task_state(tsk, TASK_UNINTERRUPTIBLE); - } - - tsk->state = TASK_RUNNING; - - return sem; -} - -/* - * wait for the read lock to be granted - */ -struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem) -{ - struct rwsem_waiter waiter; - - rwsemtrace(sem,"Entering rwsem_down_read_failed"); - - waiter.flags = RWSEM_WAITING_FOR_READ; - rwsem_down_failed_common(sem,&waiter,RWSEM_WAITING_BIAS-RWSEM_ACTIVE_BIAS); - - rwsemtrace(sem,"Leaving rwsem_down_read_failed"); - return sem; + spin_lock(&sem->lock); + } while(wait.task_list.next); } -/* - * wait for the write lock to be granted - */ -struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem) +void rwsem_wake(struct rw_semaphore *sem) { - struct rwsem_waiter waiter; + struct list_head * entry, * head = &sem->wait; + int last = 0; - rwsemtrace(sem,"Entering rwsem_down_write_failed"); + while ((entry = head->prev) != head) { + struct rwsem_wait_queue * wait; - waiter.flags = RWSEM_WAITING_FOR_WRITE; - rwsem_down_failed_common(sem,&waiter,-RWSEM_ACTIVE_BIAS); - - rwsemtrace(sem,"Leaving rwsem_down_write_failed"); - return sem; -} + wait = list_entry(entry, struct rwsem_wait_queue, task_list); -/* - * handle waking up a waiter on the semaphore - * - up_read has decremented the active part of the count if we come here - */ -struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) -{ - rwsemtrace(sem,"Entering rwsem_wake"); + if (wait->retire == RWSEM_WRITE_BLOCKING_BIAS) { + if (sem->count & RWSEM_READ_MASK) + break; + last = 1; + } - spin_lock(&sem->wait_lock); - - /* do nothing if list empty */ - if (!list_empty(&sem->wait_list)) - sem = __rwsem_do_wake(sem); - - spin_unlock(&sem->wait_lock); - - rwsemtrace(sem,"Leaving rwsem_wake"); - - return sem; + /* convert write lock into read lock when read become active */ + sem->count -= wait->retire; + list_del(entry); + entry->next = NULL; + wake_up_process(wait->task); + + if (last) + break; + } } -EXPORT_SYMBOL_NOVERS(rwsem_down_read_failed); -EXPORT_SYMBOL_NOVERS(rwsem_down_write_failed); -EXPORT_SYMBOL_NOVERS(rwsem_wake); -#if RWSEM_DEBUG -EXPORT_SYMBOL(rwsemtrace); -#endif +EXPORT_SYMBOL(rwsem_down_failed); +EXPORT_SYMBOL(rwsem_wake);