Subject: mm: pagefault_disabled() From: Peter Zijlstra Date: Thu Aug 11 15:31:31 CEST 2011 Wrap the test for pagefault_disabled() into a helper, this allows us to remove the need for current->pagefault_disabled on !-rt kernels. Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org --- arch/alpha/mm/fault.c | 2 +- arch/arm/mm/fault.c | 2 +- arch/avr32/mm/fault.c | 3 +-- arch/cris/mm/fault.c | 2 +- arch/frv/mm/fault.c | 2 +- arch/ia64/mm/fault.c | 2 +- arch/m32r/mm/fault.c | 2 +- arch/m68k/mm/fault.c | 2 +- arch/microblaze/mm/fault.c | 2 +- arch/mips/mm/fault.c | 2 +- arch/mn10300/mm/fault.c | 2 +- arch/parisc/mm/fault.c | 2 +- arch/powerpc/mm/fault.c | 2 +- arch/s390/mm/fault.c | 6 +++--- arch/score/mm/fault.c | 2 +- arch/sh/mm/fault.c | 2 +- arch/sparc/mm/fault_32.c | 2 +- arch/sparc/mm/fault_64.c | 2 +- arch/tile/mm/fault.c | 2 +- arch/um/kernel/trap.c | 2 +- arch/x86/mm/fault.c | 2 +- arch/xtensa/mm/fault.c | 2 +- include/linux/sched.h | 14 ++++++++++++++ kernel/fork.c | 2 ++ 24 files changed, 40 insertions(+), 25 deletions(-) Index: linux-stable/arch/alpha/mm/fault.c =================================================================== --- linux-stable.orig/arch/alpha/mm/fault.c +++ linux-stable/arch/alpha/mm/fault.c @@ -108,7 +108,7 @@ do_page_fault(unsigned long address, uns /* If we're in an interrupt context, or have no user context, we must not take the fault. */ - if (!mm || in_atomic() || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC Index: linux-stable/arch/arm/mm/fault.c =================================================================== --- linux-stable.orig/arch/arm/mm/fault.c +++ linux-stable/arch/arm/mm/fault.c @@ -279,7 +279,7 @@ do_page_fault(unsigned long addr, unsign * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; /* Index: linux-stable/arch/avr32/mm/fault.c =================================================================== --- linux-stable.orig/arch/avr32/mm/fault.c +++ linux-stable/arch/avr32/mm/fault.c @@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned l * If we're in an interrupt or have no user context, we must * not take the fault... */ - if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) || - current->pagefault_disabled) + if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled()) goto no_context; local_irq_enable(); Index: linux-stable/arch/cris/mm/fault.c =================================================================== --- linux-stable.orig/arch/cris/mm/fault.c +++ linux-stable/arch/cris/mm/fault.c @@ -114,7 +114,7 @@ do_page_fault(unsigned long address, str * user context, we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; retry: Index: linux-stable/arch/frv/mm/fault.c =================================================================== --- linux-stable.orig/arch/frv/mm/fault.c +++ linux-stable/arch/frv/mm/fault.c @@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datamm * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-stable/arch/ia64/mm/fault.c =================================================================== --- linux-stable.orig/arch/ia64/mm/fault.c +++ linux-stable/arch/ia64/mm/fault.c @@ -98,7 +98,7 @@ ia64_do_page_fault (unsigned long addres /* * If we're in an interrupt or have no user context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP Index: linux-stable/arch/m32r/mm/fault.c =================================================================== --- linux-stable.orig/arch/m32r/mm/fault.c +++ linux-stable/arch/m32r/mm/fault.c @@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user context or are running in an * atomic region then we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; /* When running in the kernel we expect faults to occur only to Index: linux-stable/arch/m68k/mm/fault.c =================================================================== --- linux-stable.orig/arch/m68k/mm/fault.c +++ linux-stable/arch/m68k/mm/fault.c @@ -85,7 +85,7 @@ int do_page_fault(struct pt_regs *regs, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; retry: Index: linux-stable/arch/microblaze/mm/fault.c =================================================================== --- linux-stable.orig/arch/microblaze/mm/fault.c +++ linux-stable/arch/microblaze/mm/fault.c @@ -108,7 +108,7 @@ void do_page_fault(struct pt_regs *regs, if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { if (kernel_mode(regs)) goto bad_area_nosemaphore; Index: linux-stable/arch/mips/mm/fault.c =================================================================== --- linux-stable.orig/arch/mips/mm/fault.c +++ linux-stable/arch/mips/mm/fault.c @@ -89,7 +89,7 @@ asmlinkage void __kprobes do_page_fault( * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; retry: Index: linux-stable/arch/mn10300/mm/fault.c =================================================================== --- linux-stable.orig/arch/mn10300/mm/fault.c +++ linux-stable/arch/mn10300/mm/fault.c @@ -167,7 +167,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-stable/arch/parisc/mm/fault.c =================================================================== --- linux-stable.orig/arch/parisc/mm/fault.c +++ linux-stable/arch/parisc/mm/fault.c @@ -176,7 +176,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long acc_type; int fault; - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; down_read(&mm->mmap_sem); Index: linux-stable/arch/powerpc/mm/fault.c =================================================================== --- linux-stable.orig/arch/powerpc/mm/fault.c +++ linux-stable/arch/powerpc/mm/fault.c @@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_re if (!arch_irq_disabled_regs(regs)) local_irq_enable(); - if (in_atomic() || mm == NULL || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { if (!user_mode(regs)) return SIGSEGV; /* in_atomic() in user mode is really bad, Index: linux-stable/arch/s390/mm/fault.c =================================================================== --- linux-stable.orig/arch/s390/mm/fault.c +++ linux-stable/arch/s390/mm/fault.c @@ -286,8 +286,8 @@ static inline int do_exception(struct pt * user context. */ fault = VM_FAULT_BADCONTEXT; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - tsk->pagefault_disabled)) + if (unlikely(!user_space_fault(trans_exc_code) || + !mm || pagefault_disabled())) goto out; address = trans_exc_code & __FAIL_ADDR_MASK; @@ -425,7 +425,7 @@ void __kprobes do_asce_exception(struct trans_exc_code = regs->int_parm_long; if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - current->pagefault_disabled)) + pagefault_disabled())) goto no_context; down_read(&mm->mmap_sem); Index: linux-stable/arch/score/mm/fault.c =================================================================== --- linux-stable.orig/arch/score/mm/fault.c +++ linux-stable/arch/score/mm/fault.c @@ -72,7 +72,7 @@ asmlinkage void do_page_fault(struct pt_ * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; down_read(&mm->mmap_sem); Index: linux-stable/arch/sh/mm/fault.c =================================================================== --- linux-stable.orig/arch/sh/mm/fault.c +++ linux-stable/arch/sh/mm/fault.c @@ -445,7 +445,7 @@ asmlinkage void __kprobes do_page_fault( * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } Index: linux-stable/arch/sparc/mm/fault_32.c =================================================================== --- linux-stable.orig/arch/sparc/mm/fault_32.c +++ linux-stable/arch/sparc/mm/fault_32.c @@ -200,7 +200,7 @@ asmlinkage void do_sparc_fault(struct pt * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); Index: linux-stable/arch/sparc/mm/fault_64.c =================================================================== --- linux-stable.orig/arch/sparc/mm/fault_64.c +++ linux-stable/arch/sparc/mm/fault_64.c @@ -323,7 +323,7 @@ asmlinkage void __kprobes do_sparc64_fau * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_enabled) + if (!mm || pagefault_disabled()) goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); Index: linux-stable/arch/tile/mm/fault.c =================================================================== --- linux-stable.orig/arch/tile/mm/fault.c +++ linux-stable/arch/tile/mm/fault.c @@ -359,7 +359,7 @@ static int handle_page_fault(struct pt_r * If we're in an interrupt, have no user context or are running in an * atomic region then we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } Index: linux-stable/arch/um/kernel/trap.c =================================================================== --- linux-stable.orig/arch/um/kernel/trap.c +++ linux-stable/arch/um/kernel/trap.c @@ -39,7 +39,7 @@ int handle_page_fault(unsigned long addr * If the fault was during atomic operation, don't take the fault, just * fail. */ - if (in_atomic() || current->pagefault_disabled) + if (pagefault_disabled()) goto out_nosemaphore; retry: Index: linux-stable/arch/x86/mm/fault.c =================================================================== --- linux-stable.orig/arch/x86/mm/fault.c +++ linux-stable/arch/x86/mm/fault.c @@ -1094,7 +1094,7 @@ do_page_fault(struct pt_regs *regs, unsi * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } Index: linux-stable/arch/xtensa/mm/fault.c =================================================================== --- linux-stable.orig/arch/xtensa/mm/fault.c +++ linux-stable/arch/xtensa/mm/fault.c @@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs) /* If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { bad_page_fault(regs, address, SIGSEGV); return; } Index: linux-stable/include/linux/sched.h =================================================================== --- linux-stable.orig/include/linux/sched.h +++ linux-stable/include/linux/sched.h @@ -91,6 +91,7 @@ struct sched_param { #include #include #include +#include #include @@ -1448,7 +1449,9 @@ struct task_struct { /* mutex deadlock detection */ struct mutex_waiter *blocked_on; #endif +#ifdef CONFIG_PREEMPT_RT_FULL int pagefault_disabled; +#endif #ifdef CONFIG_TRACE_IRQFLAGS unsigned int irq_events; unsigned long hardirq_enable_ip; @@ -1600,6 +1603,17 @@ struct task_struct { /* Future-safe accessor for struct task_struct's cpus_allowed. */ #define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +#ifdef CONFIG_PREEMPT_RT_FULL +static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; } +#else +static inline bool cur_pf_disabled(void) { return false; } +#endif + +static inline bool pagefault_disabled(void) +{ + return in_atomic() || cur_pf_disabled(); +} + /* * Priority of a process goes from 0..MAX_PRIO-1, valid RT * priority is 0..MAX_RT_PRIO-1, and SCHED_NORMAL/SCHED_BATCH Index: linux-stable/kernel/fork.c =================================================================== --- linux-stable.orig/kernel/fork.c +++ linux-stable/kernel/fork.c @@ -1298,7 +1298,9 @@ static struct task_struct *copy_process( p->hardirq_context = 0; p->softirq_context = 0; #endif +#ifdef CONFIG_PREEMPT_RT_FULL p->pagefault_disabled = 0; +#endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0;