From: Keith Owens As requested by Linus, update all architectures to add the common infrastructure. Tested on ia64 and i386. Enable interrupts while waiting for a disabled spinlock, but only if interrupts were enabled before issuing spin_lock_irqsave(). This patch consists of three sections :- * An architecture independent change to call _raw_spin_lock_flags() instead of _raw_spin_lock() when the flags are available. * An ia64 specific change to implement _raw_spin_lock_flags() and to define _raw_spin_lock(lock) as _raw_spin_lock_flags(lock, 0) for the ASM_SUPPORTED case. * Patches for all other architectures and for ia64 with !ASM_SUPPORTED to map _raw_spin_lock_flags(lock, flags) to _raw_spin_lock(lock). Architecture maintainers can define _raw_spin_lock_flags() to do something useful if they want to enable interrupts while waiting for a disabled spinlock. --- 25-akpm/arch/ia64/kernel/head.S | 22 +++++++++++++++++----- 25-akpm/include/asm-alpha/spinlock.h | 1 + 25-akpm/include/asm-arm/spinlock.h | 1 + 25-akpm/include/asm-i386/spinlock.h | 1 + 25-akpm/include/asm-ia64/spinlock.h | 24 +++++++++++++++--------- 25-akpm/include/asm-mips/spinlock.h | 1 + 25-akpm/include/asm-parisc/spinlock.h | 1 + 25-akpm/include/asm-ppc/spinlock.h | 1 + 25-akpm/include/asm-ppc64/spinlock.h | 1 + 25-akpm/include/asm-s390/spinlock.h | 1 + 25-akpm/include/asm-sh/spinlock.h | 1 + 25-akpm/include/asm-sparc/spinlock.h | 2 ++ 25-akpm/include/asm-sparc64/spinlock.h | 2 ++ 25-akpm/include/asm-x86_64/spinlock.h | 1 + 25-akpm/include/linux/spinlock.h | 2 +- 15 files changed, 47 insertions(+), 15 deletions(-) diff -puN arch/ia64/kernel/head.S~allow-architectures-to-reenable-interrupts-on-contended-spinlocks arch/ia64/kernel/head.S --- 25/arch/ia64/kernel/head.S~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.678377240 -0700 +++ 25-akpm/arch/ia64/kernel/head.S 2004-04-27 21:19:55.702373592 -0700 @@ -866,12 +866,14 @@ SET_REG(b5); * Inputs: * ar.pfs - saved CFM of caller * ar.ccv - 0 (and available for use) + * r27 - flags from spin_lock_irqsave or 0. Must be preserved. * r28 - available for use. * r29 - available for use. * r30 - available for use. * r31 - address of lock, available for use. * b6 - return address * p14 - available for use. + * p15 - used to track flag status. * * If you patch this code to use more registers, do not forget to update * the clobber lists for spin_lock() in include/asm-ia64/spinlock.h. @@ -885,22 +887,26 @@ GLOBAL_ENTRY(ia64_spinlock_contention_pr .save rp, r28 .body nop 0 - nop 0 + tbit.nz p15,p0=r27,IA64_PSR_I_BIT .restore sp // pop existing prologue after next insn mov b6 = r28 .prologue .save ar.pfs, r0 .altrp b6 .body + ;; +(p15) ssm psr.i // reenable interrupts if they were on + // DavidM says that srlz.d is slow and is not required in this case .wait: // exponential backoff, kdb, lockmeter etc. go in here hint @pause ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word nop 0 ;; - cmp4.eq p14,p0=r30,r0 -(p14) br.cond.sptk.few b6 // lock is now free, try to acquire - br.cond.sptk.few .wait + cmp4.ne p14,p0=r30,r0 +(p14) br.cond.sptk.few .wait +(p15) rsm psr.i // disable interrupts if we reenabled them + br.cond.sptk.few b6 // lock is now free, try to acquire END(ia64_spinlock_contention_pre3_4) #else @@ -909,14 +915,20 @@ GLOBAL_ENTRY(ia64_spinlock_contention) .prologue .altrp b6 .body + tbit.nz p15,p0=r27,IA64_PSR_I_BIT + ;; .wait: +(p15) ssm psr.i // reenable interrupts if they were on + // DavidM says that srlz.d is slow and is not required in this case +.wait2: // exponential backoff, kdb, lockmeter etc. go in here hint @pause ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word ;; cmp4.ne p14,p0=r30,r0 mov r30 = 1 -(p14) br.cond.sptk.few .wait +(p14) br.cond.sptk.few .wait2 +(p15) rsm psr.i // disable interrupts if we reenabled them ;; cmpxchg4.acq r30=[r31], r30, ar.ccv ;; diff -puN include/asm-alpha/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-alpha/spinlock.h --- 25/include/asm-alpha/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.680376936 -0700 +++ 25-akpm/include/asm-alpha/spinlock.h 2004-04-27 21:19:55.703373440 -0700 @@ -40,6 +40,7 @@ typedef struct { #define spin_is_locked(x) ((x)->lock != 0) #define spin_unlock_wait(x) ({ do { barrier(); } while ((x)->lock); }) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) #ifdef CONFIG_DEBUG_SPINLOCK extern void _raw_spin_unlock(spinlock_t * lock); diff -puN include/asm-arm/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-arm/spinlock.h --- 25/include/asm-arm/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.681376784 -0700 +++ 25-akpm/include/asm-arm/spinlock.h 2004-04-27 21:19:55.703373440 -0700 @@ -24,6 +24,7 @@ typedef struct { #define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while (0) #define spin_is_locked(x) ((x)->lock != 0) #define spin_unlock_wait(x) do { barrier(); } while (spin_is_locked(x)) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) static inline void _raw_spin_lock(spinlock_t *lock) { diff -puN include/asm-i386/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-i386/spinlock.h --- 25/include/asm-i386/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.682376632 -0700 +++ 25-akpm/include/asm-i386/spinlock.h 2004-04-27 21:19:55.704373288 -0700 @@ -42,6 +42,7 @@ typedef struct { #define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0) #define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x)) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) #ifdef CONFIG_SPINLINE diff -puN include/asm-ia64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ia64/spinlock.h --- 25/include/asm-ia64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.684376328 -0700 +++ 25-akpm/include/asm-ia64/spinlock.h 2004-04-27 21:19:55.701373744 -0700 @@ -32,10 +32,10 @@ typedef struct { * carefully coded to touch only those registers that spin_lock() marks "clobbered". */ -#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "r28", "r29", "r30", "b6", "memory" +#define IA64_SPINLOCK_CLOBBERS "ar.ccv", "ar.pfs", "p14", "p15", "r27", "r28", "r29", "r30", "b6", "memory" static inline void -_raw_spin_lock (spinlock_t *lock) +_raw_spin_lock_flags (spinlock_t *lock, unsigned long flags) { register volatile unsigned int *ptr asm ("r31") = &lock->lock; @@ -50,9 +50,10 @@ _raw_spin_lock (spinlock_t *lock) "cmpxchg4.acq r30 = [%1], r30, ar.ccv\n\t" "movl r29 = ia64_spinlock_contention_pre3_4;;\n\t" "cmp4.ne p14, p0 = r30, r0\n\t" - "mov b6 = r29;;\n" + "mov b6 = r29;;\n\t" + "mov r27=%2\n\t" "(p14) br.cond.spnt.many b6" - : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS); + : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS); # else asm volatile ("{\n\t" " mov ar.ccv = r0\n\t" @@ -60,33 +61,38 @@ _raw_spin_lock (spinlock_t *lock) " mov r30 = 1;;\n\t" "}\n\t" "cmpxchg4.acq r30 = [%1], r30, ar.ccv;;\n\t" - "cmp4.ne p14, p0 = r30, r0\n" + "cmp4.ne p14, p0 = r30, r0\n\t" + "mov r27=%2\n\t" "(p14) brl.cond.spnt.many ia64_spinlock_contention_pre3_4;;" - : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS); + : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS); # endif /* CONFIG_MCKINLEY */ #else # ifdef CONFIG_ITANIUM /* don't use brl on Itanium... */ /* mis-declare, so we get the entry-point, not it's function descriptor: */ asm volatile ("mov r30 = 1\n\t" + "mov r27=%2\n\t" "mov ar.ccv = r0;;\n\t" "cmpxchg4.acq r30 = [%0], r30, ar.ccv\n\t" "movl r29 = ia64_spinlock_contention;;\n\t" "cmp4.ne p14, p0 = r30, r0\n\t" - "mov b6 = r29;;\n" + "mov b6 = r29;;\n\t" "(p14) br.call.spnt.many b6 = b6" - : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS); + : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS); # else asm volatile ("mov r30 = 1\n\t" + "mov r27=%2\n\t" "mov ar.ccv = r0;;\n\t" "cmpxchg4.acq r30 = [%0], r30, ar.ccv;;\n\t" "cmp4.ne p14, p0 = r30, r0\n\t" "(p14) brl.call.spnt.many b6=ia64_spinlock_contention;;" - : "=r"(ptr) : "r"(ptr) : IA64_SPINLOCK_CLOBBERS); + : "=r"(ptr) : "r"(ptr), "r" (flags) : IA64_SPINLOCK_CLOBBERS); # endif /* CONFIG_MCKINLEY */ #endif } +#define _raw_spin_lock(lock) _raw_spin_lock_flags(lock, 0) #else /* !ASM_SUPPORTED */ +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) # define _raw_spin_lock(x) \ do { \ __u32 *ia64_spinlock_ptr = (__u32 *) (x); \ diff -puN include/asm-mips/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-mips/spinlock.h --- 25/include/asm-mips/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.685376176 -0700 +++ 25-akpm/include/asm-mips/spinlock.h 2004-04-27 21:19:55.704373288 -0700 @@ -23,6 +23,7 @@ typedef struct { #define spin_is_locked(x) ((x)->lock != 0) #define spin_unlock_wait(x) do { barrier(); } while ((x)->lock) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) /* * Simple spin lock operations. There are two variants, one clears IRQ's diff -puN include/asm-parisc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-parisc/spinlock.h --- 25/include/asm-parisc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.687375872 -0700 +++ 25-akpm/include/asm-parisc/spinlock.h 2004-04-27 21:19:55.705373136 -0700 @@ -15,6 +15,7 @@ #define spin_is_locked(x) ((x)->lock == 0) #define spin_unlock_wait(x) do { barrier(); } while(((volatile spinlock_t *)(x))->lock == 0) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) #if 1 #define _raw_spin_lock(x) do { \ diff -puN include/asm-ppc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ppc64/spinlock.h --- 25/include/asm-ppc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.688375720 -0700 +++ 25-akpm/include/asm-ppc64/spinlock.h 2004-04-27 21:19:55.705373136 -0700 @@ -22,6 +22,7 @@ typedef struct { #define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 } #define spin_is_locked(x) ((x)->lock != 0) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) static __inline__ int _raw_spin_trylock(spinlock_t *lock) { diff -puN include/asm-ppc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-ppc/spinlock.h --- 25/include/asm-ppc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.689375568 -0700 +++ 25-akpm/include/asm-ppc/spinlock.h 2004-04-27 21:19:55.705373136 -0700 @@ -27,6 +27,7 @@ typedef struct { #define spin_lock_init(x) do { *(x) = SPIN_LOCK_UNLOCKED; } while(0) #define spin_is_locked(x) ((x)->lock != 0) #define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x)) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) #ifndef CONFIG_DEBUG_SPINLOCK diff -puN include/asm-s390/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-s390/spinlock.h --- 25/include/asm-s390/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.690375416 -0700 +++ 25-akpm/include/asm-s390/spinlock.h 2004-04-27 21:19:55.706372984 -0700 @@ -42,6 +42,7 @@ typedef struct { #define spin_lock_init(lp) do { (lp)->lock = 0; } while(0) #define spin_unlock_wait(lp) do { barrier(); } while(((volatile spinlock_t *)(lp))->lock) #define spin_is_locked(x) ((x)->lock != 0) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) extern inline void _raw_spin_lock(spinlock_t *lp) { diff -puN include/asm-sh/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sh/spinlock.h --- 25/include/asm-sh/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.691375264 -0700 +++ 25-akpm/include/asm-sh/spinlock.h 2004-04-27 21:19:55.706372984 -0700 @@ -25,6 +25,7 @@ typedef struct { #define spin_is_locked(x) ((x)->lock != 0) #define spin_unlock_wait(x) do { barrier(); } while (spin_is_locked(x)) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) /* * Simple spin lock operations. There are two variants, one clears IRQ's diff -puN include/asm-sparc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sparc64/spinlock.h --- 25/include/asm-sparc64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.693374960 -0700 +++ 25-akpm/include/asm-sparc64/spinlock.h 2004-04-27 21:19:55.707372832 -0700 @@ -114,6 +114,8 @@ extern int _spin_trylock (spinlock_t *lo #endif /* CONFIG_DEBUG_SPINLOCK */ +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) + /* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */ #ifndef CONFIG_DEBUG_SPINLOCK diff -puN include/asm-sparc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-sparc/spinlock.h --- 25/include/asm-sparc/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.694374808 -0700 +++ 25-akpm/include/asm-sparc/spinlock.h 2004-04-27 21:19:55.707372832 -0700 @@ -216,6 +216,8 @@ extern __inline__ void _raw_write_lock(r #endif /* CONFIG_DEBUG_SPINLOCK */ +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) + #endif /* !(__ASSEMBLY__) */ #endif /* __SPARC_SPINLOCK_H */ diff -puN include/asm-x86_64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/asm-x86_64/spinlock.h --- 25/include/asm-x86_64/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.695374656 -0700 +++ 25-akpm/include/asm-x86_64/spinlock.h 2004-04-27 21:19:55.708372680 -0700 @@ -41,6 +41,7 @@ typedef struct { #define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0) #define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x)) +#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock) #define spin_lock_string \ "\n1:\t" \ diff -puN include/linux/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks include/linux/spinlock.h --- 25/include/linux/spinlock.h~allow-architectures-to-reenable-interrupts-on-contended-spinlocks 2004-04-27 21:19:55.697374352 -0700 +++ 25-akpm/include/linux/spinlock.h 2004-04-27 21:19:55.701373744 -0700 @@ -280,7 +280,7 @@ do { \ do { \ local_irq_save(flags); \ preempt_disable(); \ - _raw_spin_lock(lock); \ + _raw_spin_lock_flags(lock, flags); \ } while (0) #define spin_lock_irq(lock) \ _