diff -urN rwsem-recurse-ref/arch/alpha/mm/fault.c rwsem-recurse/arch/alpha/mm/fault.c --- rwsem-recurse-ref/arch/alpha/mm/fault.c Sun Sep 23 21:11:28 2001 +++ rwsem-recurse/arch/alpha/mm/fault.c Mon Nov 12 01:17:19 2001 @@ -113,7 +113,7 @@ goto vmalloc_fault; #endif - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -147,7 +147,7 @@ * the fault. */ fault = handle_mm_fault(mm, vma, address, cause > 0); - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (fault < 0) goto out_of_memory; @@ -161,7 +161,7 @@ * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (user_mode(regs)) { force_sig(SIGSEGV, current); @@ -198,7 +198,7 @@ if (current->pid == 1) { current->policy |= SCHED_YIELD; schedule(); - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); goto survive; } printk(KERN_ALERT "VM: killing process %s(%d)\n", diff -urN rwsem-recurse-ref/arch/i386/mm/fault.c rwsem-recurse/arch/i386/mm/fault.c --- rwsem-recurse-ref/arch/i386/mm/fault.c Wed Oct 10 02:14:56 2001 +++ rwsem-recurse/arch/i386/mm/fault.c Mon Nov 12 01:17:19 2001 @@ -192,7 +192,7 @@ if (in_interrupt() || !mm) goto no_context; - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); vma = find_vma(mm, address); if (!vma) @@ -266,7 +266,7 @@ if (bit < 32) tsk->thread.screen_bitmap |= 1 << bit; } - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); return; /* @@ -274,7 +274,7 @@ * Fix it, but check if it's kernel or user first.. */ bad_area: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); /* User mode accesses just cause a SIGSEGV */ if (error_code & 4) { @@ -342,11 +342,11 @@ * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (tsk->pid == 1) { tsk->policy |= SCHED_YIELD; schedule(); - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); goto survive; } printk("VM: killing process %s\n", tsk->comm); @@ -355,7 +355,7 @@ goto no_context; do_sigbus: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); /* * Send a sigbus, regardless of whether we were in kernel diff -urN rwsem-recurse-ref/arch/ia64/mm/fault.c rwsem-recurse/arch/ia64/mm/fault.c --- rwsem-recurse-ref/arch/ia64/mm/fault.c Sat Nov 10 02:04:27 2001 +++ rwsem-recurse/arch/ia64/mm/fault.c Mon Nov 12 01:17:50 2001 @@ -59,7 +59,7 @@ if (in_interrupt() || !mm) goto no_context; - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); vma = find_vma_prev(mm, address, &prev_vma); if (!vma) @@ -114,7 +114,7 @@ default: goto out_of_memory; } - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); return; check_expansion: @@ -137,7 +137,7 @@ goto good_area; bad_area: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (isr & IA64_ISR_SP) { /* * This fault was due to a speculative load set the "ed" bit in the psr to @@ -192,7 +192,7 @@ return; out_of_memory: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (current->pid == 1) { current->policy |= SCHED_YIELD; schedule(); diff -urN rwsem-recurse-ref/arch/ppc/mm/fault.c rwsem-recurse/arch/ppc/mm/fault.c --- rwsem-recurse-ref/arch/ppc/mm/fault.c Wed Oct 10 02:14:58 2001 +++ rwsem-recurse/arch/ppc/mm/fault.c Mon Nov 12 01:17:19 2001 @@ -103,7 +103,7 @@ bad_page_fault(regs, address, SIGSEGV); return; } - down_read(&mm->mmap_sem); + down_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); vma = find_vma(mm, address); if (!vma) goto bad_area; @@ -164,7 +164,7 @@ goto out_of_memory; } - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); /* * keep track of tlb+htab misses that are good addrs but * just need pte's created via handle_mm_fault() @@ -174,7 +174,7 @@ return; bad_area: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); pte_errors++; /* User mode accesses cause a SIGSEGV */ @@ -195,7 +195,7 @@ * us unable to handle the page fault gracefully. */ out_of_memory: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); if (current->pid == 1) { current->policy |= SCHED_YIELD; schedule(); @@ -209,7 +209,7 @@ return; do_sigbus: - up_read(&mm->mmap_sem); + up_read_recursive(&mm->mmap_sem, ¤t->mm_recursor); info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRERR; diff -urN rwsem-recurse-ref/fs/exec.c rwsem-recurse/fs/exec.c --- rwsem-recurse-ref/fs/exec.c Tue Nov 6 02:04:50 2001 +++ rwsem-recurse/fs/exec.c Mon Nov 12 01:17:19 2001 @@ -973,9 +973,9 @@ if (do_truncate(file->f_dentry, 0) != 0) goto close_fail; - down_read(¤t->mm->mmap_sem); + down_read_recursive(¤t->mm->mmap_sem, ¤t->mm_recursor); retval = binfmt->core_dump(signr, regs, file); - up_read(¤t->mm->mmap_sem); + up_read_recursive(¤t->mm->mmap_sem, ¤t->mm_recursor); close_fail: filp_close(file, NULL); diff -urN rwsem-recurse-ref/include/linux/rwsem.h rwsem-recurse/include/linux/rwsem.h --- rwsem-recurse-ref/include/linux/rwsem.h Mon Nov 12 01:17:09 2001 +++ rwsem-recurse/include/linux/rwsem.h Mon Nov 12 01:17:19 2001 @@ -18,6 +18,11 @@ #endif }; +struct rw_sem_recursor +{ + int counter; +}; + #if RWSEM_DEBUG #define __SEM_DEBUG_INIT(name) \ , (long)&(name).__magic @@ -42,6 +47,7 @@ __SEM_DEBUG_INIT(name) \ } #define RWSEM_INITIALIZER(name) __RWSEM_INITIALIZER(name, 0) +#define RWSEM_RECURSOR_INITIALIZER ((struct rw_sem_recursor) { 0, }) #define __DECLARE_RWSEM(name, count) \ struct rw_semaphore name = __RWSEM_INITIALIZER(name, count) @@ -112,6 +118,34 @@ spin_lock(&sem->lock); sem->count -= RWSEM_WRITE_BIAS; if (unlikely(sem->count)) + rwsem_wake(sem); + spin_unlock(&sem->lock); +} + +static inline void down_read_recursive(struct rw_semaphore *sem, + struct rw_sem_recursor * recursor) +{ + int count, counter; + CHECK_MAGIC(sem->__magic); + + spin_lock(&sem->lock); + count = sem->count; + sem->count += RWSEM_READ_BIAS; + counter = recursor->counter++; + if (unlikely(count < 0 && !counter && !(count & RWSEM_READ_MASK))) + rwsem_down_failed(sem, RWSEM_READ_BLOCKING_BIAS); + spin_unlock(&sem->lock); +} + +static inline void up_read_recursive(struct rw_semaphore *sem, + struct rw_sem_recursor * recursor) +{ + CHECK_MAGIC(sem->__magic); + + spin_lock(&sem->lock); + sem->count -= RWSEM_READ_BIAS; + recursor->counter--; + if (unlikely(sem->count < 0 && !(sem->count & RWSEM_READ_MASK))) rwsem_wake(sem); spin_unlock(&sem->lock); } diff -urN rwsem-recurse-ref/include/linux/sched.h rwsem-recurse/include/linux/sched.h --- rwsem-recurse-ref/include/linux/sched.h Mon Nov 12 01:17:09 2001 +++ rwsem-recurse/include/linux/sched.h Mon Nov 12 01:17:19 2001 @@ -316,6 +316,7 @@ struct task_struct *next_task, *prev_task; struct mm_struct *active_mm; + struct rw_sem_recursor mm_recursor; struct list_head local_pages; unsigned int allocation_order, nr_local_pages; @@ -465,6 +466,7 @@ policy: SCHED_OTHER, \ mm: NULL, \ active_mm: &init_mm, \ + mm_recursor: RWSEM_RECURSOR_INITIALIZER, \ cpus_allowed: -1, \ run_list: LIST_HEAD_INIT(tsk.run_list), \ next_task: &tsk, \