From: Christoph Lameter This patch extracts all the operations on counters protected by the page table lock (currently rss and anon_rss) into definitions in include/linux/sched.h. All rss operations are performed through the following macros: get_mm_counter(mm, member) -> Obtain the value of a counter set_mm_counter(mm, member, value) -> Set the value of a counter update_mm_counter(mm, member, value) -> Add to a counter inc_mm_counter(mm, member) -> Increment a counter dec_mm_counter(mm, member) -> Decrement a counter With this patch it becomes easier to add new counters and it is possible to redefine the method of counter handling. The counters are an issue for scalability since they are used in frequently used code paths and may cause cache line bouncing. F.e. One may not use counters at all and count the pages when needed, switch to atomic operations if the mm_struct locking changes or split the rss into counters that can be locally incremented. The relevant fields of the task_struct are renamed with a leading underscore to catch out people who are not using the acceessor macros. Signed-off-by: Christoph Lameter Signed-off-by: Andrew Morton --- 25-akpm/arch/i386/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/ia64/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/m68k/atari/stram.c | 2 +- 25-akpm/arch/mips/kernel/irixelf.c | 2 +- 25-akpm/arch/ppc64/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/sh/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/sh64/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/sparc64/kernel/binfmt_aout32.c | 2 +- 25-akpm/arch/sparc64/mm/hugetlbpage.c | 6 +++--- 25-akpm/arch/x86_64/ia32/ia32_aout.c | 2 +- 25-akpm/fs/binfmt_aout.c | 2 +- 25-akpm/fs/binfmt_elf.c | 2 +- 25-akpm/fs/binfmt_elf_fdpic.c | 2 +- 25-akpm/fs/binfmt_flat.c | 2 +- 25-akpm/fs/binfmt_som.c | 2 +- 25-akpm/fs/exec.c | 2 +- 25-akpm/fs/proc/array.c | 2 +- 25-akpm/fs/proc/task_mmu.c | 8 +++++--- 25-akpm/include/asm-arm/tlb.h | 4 ++-- 25-akpm/include/asm-arm26/tlb.h | 4 ++-- 25-akpm/include/asm-generic/tlb.h | 4 ++-- 25-akpm/include/asm-ia64/tlb.h | 4 ++-- 25-akpm/include/asm-sparc64/tlb.h | 4 ++-- 25-akpm/include/linux/sched.h | 15 +++++++++++++-- 25-akpm/kernel/acct.c | 2 +- 25-akpm/kernel/fork.c | 6 +++--- 25-akpm/mm/fremap.c | 4 ++-- 25-akpm/mm/memory.c | 22 ++++++++++++---------- 25-akpm/mm/mmap.c | 2 +- 25-akpm/mm/nommu.c | 6 ++++-- 25-akpm/mm/rmap.c | 14 +++++++------- 25-akpm/mm/swapfile.c | 2 +- 32 files changed, 88 insertions(+), 71 deletions(-) diff -puN arch/i386/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/i386/mm/hugetlbpage.c --- 25/arch/i386/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/i386/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -46,7 +46,7 @@ static void set_huge_pte(struct mm_struc { pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) { entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); @@ -86,7 +86,7 @@ int copy_hugetlb_page_range(struct mm_st ptepage = pte_page(entry); get_page(ptepage); set_pte(dst_pte, entry); - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); addr += HPAGE_SIZE; } return 0; @@ -222,7 +222,7 @@ void unmap_hugepage_range(struct vm_area page = pte_page(pte); put_page(page); } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm ,rss, -((end - start) >> PAGE_SHIFT)); flush_tlb_range(vma, start, end); } diff -puN arch/ia64/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/ia64/mm/hugetlbpage.c --- 25/arch/ia64/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/ia64/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -73,7 +73,7 @@ set_huge_pte (struct mm_struct *mm, stru { pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) { entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); @@ -116,7 +116,7 @@ int copy_hugetlb_page_range(struct mm_st ptepage = pte_page(entry); get_page(ptepage); set_pte(dst_pte, entry); - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); addr += HPAGE_SIZE; } return 0; @@ -246,7 +246,7 @@ void unmap_hugepage_range(struct vm_area put_page(page); pte_clear(mm, address, pte); } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm, rss, - ((end - start) >> PAGE_SHIFT)); flush_tlb_range(vma, start, end); } diff -puN arch/m68k/atari/stram.c~mm-counter-operations-through-macros arch/m68k/atari/stram.c --- 25/arch/m68k/atari/stram.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/m68k/atari/stram.c 2005-03-14 21:38:16.000000000 -0800 @@ -635,7 +635,7 @@ static inline void unswap_pte(struct vm_ set_pte(dir, pte_mkdirty(mk_pte(page, vma->vm_page_prot))); swap_free(entry); get_page(page); - ++vma->vm_mm->rss; + inc_mm_counter(vma->vm_mm, rss); } static inline void unswap_pmd(struct vm_area_struct * vma, pmd_t *dir, diff -puN arch/mips/kernel/irixelf.c~mm-counter-operations-through-macros arch/mips/kernel/irixelf.c --- 25/arch/mips/kernel/irixelf.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/mips/kernel/irixelf.c 2005-03-14 21:38:16.000000000 -0800 @@ -692,7 +692,7 @@ static int load_irix_binary(struct linux /* Do this so that we can load the interpreter, if need be. We will * change some of these later. */ - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT); current->mm->start_stack = bprm->p; diff -puN arch/ppc64/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/ppc64/mm/hugetlbpage.c --- 25/arch/ppc64/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/ppc64/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -154,7 +154,7 @@ static void set_huge_pte(struct mm_struc { pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) { entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); @@ -316,7 +316,7 @@ int copy_hugetlb_page_range(struct mm_st ptepage = pte_page(entry); get_page(ptepage); - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); set_pte_at(dst, addr, dst_pte, entry); addr += HPAGE_SIZE; @@ -426,7 +426,7 @@ void unmap_hugepage_range(struct vm_area put_page(page); } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); flush_tlb_pending(); } diff -puN arch/sh64/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/sh64/mm/hugetlbpage.c --- 25/arch/sh64/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/sh64/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -62,7 +62,7 @@ static void set_huge_pte(struct mm_struc unsigned long i; pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) entry = pte_mkwrite(pte_mkdirty(mk_pte(page, @@ -115,7 +115,7 @@ int copy_hugetlb_page_range(struct mm_st pte_val(entry) += PAGE_SIZE; dst_pte++; } - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); addr += HPAGE_SIZE; } return 0; @@ -206,7 +206,7 @@ void unmap_hugepage_range(struct vm_area pte++; } } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); flush_tlb_range(vma, start, end); } diff -puN arch/sh/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/sh/mm/hugetlbpage.c --- 25/arch/sh/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/sh/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -62,7 +62,7 @@ static void set_huge_pte(struct mm_struc unsigned long i; pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) entry = pte_mkwrite(pte_mkdirty(mk_pte(page, @@ -115,7 +115,7 @@ int copy_hugetlb_page_range(struct mm_st pte_val(entry) += PAGE_SIZE; dst_pte++; } - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); addr += HPAGE_SIZE; } return 0; @@ -206,7 +206,7 @@ void unmap_hugepage_range(struct vm_area pte++; } } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); flush_tlb_range(vma, start, end); } diff -puN arch/sparc64/kernel/binfmt_aout32.c~mm-counter-operations-through-macros arch/sparc64/kernel/binfmt_aout32.c --- 25/arch/sparc64/kernel/binfmt_aout32.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/sparc64/kernel/binfmt_aout32.c 2005-03-14 21:38:16.000000000 -0800 @@ -241,7 +241,7 @@ static int load_aout32_binary(struct lin current->mm->brk = ex.a_bss + (current->mm->start_brk = N_BSSADDR(ex)); - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); current->mm->mmap = NULL; compute_creds(bprm); current->flags &= ~PF_FORKNOEXEC; diff -puN arch/sparc64/mm/hugetlbpage.c~mm-counter-operations-through-macros arch/sparc64/mm/hugetlbpage.c --- 25/arch/sparc64/mm/hugetlbpage.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/sparc64/mm/hugetlbpage.c 2005-03-14 21:38:16.000000000 -0800 @@ -68,7 +68,7 @@ static void set_huge_pte(struct mm_struc unsigned long i; pte_t entry; - mm->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE); if (write_access) entry = pte_mkwrite(pte_mkdirty(mk_pte(page, @@ -123,7 +123,7 @@ int copy_hugetlb_page_range(struct mm_st dst_pte++; addr += PAGE_SIZE; } - dst->rss += (HPAGE_SIZE / PAGE_SIZE); + add_mm_counter(dst, rss, HPAGE_SIZE / PAGE_SIZE); } return 0; @@ -213,7 +213,7 @@ void unmap_hugepage_range(struct vm_area pte++; } } - mm->rss -= (end - start) >> PAGE_SHIFT; + add_mm_counter(mm, rss, -((end - start) >> PAGE_SHIFT)); flush_tlb_range(vma, start, end); } diff -puN arch/x86_64/ia32/ia32_aout.c~mm-counter-operations-through-macros arch/x86_64/ia32/ia32_aout.c --- 25/arch/x86_64/ia32/ia32_aout.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/arch/x86_64/ia32/ia32_aout.c 2005-03-14 21:38:16.000000000 -0800 @@ -313,7 +313,7 @@ static int load_aout_binary(struct linux (current->mm->start_brk = N_BSSADDR(ex)); current->mm->free_area_cache = TASK_UNMAPPED_BASE; - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); current->mm->mmap = NULL; compute_creds(bprm); current->flags &= ~PF_FORKNOEXEC; diff -puN fs/binfmt_aout.c~mm-counter-operations-through-macros fs/binfmt_aout.c --- 25/fs/binfmt_aout.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/binfmt_aout.c 2005-03-14 21:38:16.000000000 -0800 @@ -317,7 +317,7 @@ static int load_aout_binary(struct linux (current->mm->start_brk = N_BSSADDR(ex)); current->mm->free_area_cache = current->mm->mmap_base; - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); current->mm->mmap = NULL; compute_creds(bprm); current->flags &= ~PF_FORKNOEXEC; diff -puN fs/binfmt_elf.c~mm-counter-operations-through-macros fs/binfmt_elf.c --- 25/fs/binfmt_elf.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/binfmt_elf.c 2005-03-14 21:38:16.000000000 -0800 @@ -773,7 +773,7 @@ static int load_elf_binary(struct linux_ /* Do this so that we can load the interpreter, if need be. We will change some of these later */ - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); current->mm->free_area_cache = current->mm->mmap_base; retval = setup_arg_pages(bprm, randomize_stack_top(STACK_TOP), executable_stack); diff -puN fs/binfmt_elf_fdpic.c~mm-counter-operations-through-macros fs/binfmt_elf_fdpic.c --- 25/fs/binfmt_elf_fdpic.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/binfmt_elf_fdpic.c 2005-03-14 21:38:16.000000000 -0800 @@ -299,7 +299,7 @@ static int load_elf_fdpic_binary(struct /* do this so that we can load the interpreter, if need be * - we will change some of these later */ - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); #ifdef CONFIG_MMU retval = setup_arg_pages(bprm, current->mm->start_stack, executable_stack); diff -puN fs/binfmt_flat.c~mm-counter-operations-through-macros fs/binfmt_flat.c --- 25/fs/binfmt_flat.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/binfmt_flat.c 2005-03-14 21:38:16.000000000 -0800 @@ -650,7 +650,7 @@ static int load_flat_file(struct linux_b current->mm->start_brk = datapos + data_len + bss_len; current->mm->brk = (current->mm->start_brk + 3) & ~3; current->mm->context.end_brk = memp + ksize((void *) memp) - stack_len; - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); } if (flags & FLAT_FLAG_KTRACE) diff -puN fs/binfmt_som.c~mm-counter-operations-through-macros fs/binfmt_som.c --- 25/fs/binfmt_som.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/binfmt_som.c 2005-03-14 21:38:16.000000000 -0800 @@ -259,7 +259,7 @@ load_som_binary(struct linux_binprm * bp create_som_tables(bprm); current->mm->start_stack = bprm->p; - current->mm->rss = 0; + set_mm_counter(current->mm, rss, 0); #if 0 printk("(start_brk) %08lx\n" , (unsigned long) current->mm->start_brk); diff -puN fs/exec.c~mm-counter-operations-through-macros fs/exec.c --- 25/fs/exec.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/exec.c 2005-03-14 21:38:16.000000000 -0800 @@ -326,7 +326,7 @@ void install_arg_page(struct vm_area_str pte_unmap(pte); goto out; } - mm->rss++; + inc_mm_counter(mm, rss); lru_cache_add_active(page); set_pte_at(mm, address, pte, pte_mkdirty(pte_mkwrite(mk_pte( page, vma->vm_page_prot)))); diff -puN fs/proc/array.c~mm-counter-operations-through-macros fs/proc/array.c --- 25/fs/proc/array.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/proc/array.c 2005-03-14 21:38:16.000000000 -0800 @@ -432,7 +432,7 @@ static int do_task_stat(struct task_stru jiffies_to_clock_t(it_real_value), start_time, vsize, - mm ? mm->rss : 0, /* you might want to shift this left 3 */ + mm ? get_mm_counter(mm, rss) : 0, /* you might want to shift this left 3 */ rsslim, mm ? mm->start_code : 0, mm ? mm->end_code : 0, diff -puN fs/proc/task_mmu.c~mm-counter-operations-through-macros fs/proc/task_mmu.c --- 25/fs/proc/task_mmu.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/fs/proc/task_mmu.c 2005-03-14 21:38:16.000000000 -0800 @@ -24,7 +24,7 @@ char *task_mem(struct mm_struct *mm, cha "VmPTE:\t%8lu kB\n", (mm->total_vm - mm->reserved_vm) << (PAGE_SHIFT-10), mm->locked_vm << (PAGE_SHIFT-10), - mm->rss << (PAGE_SHIFT-10), + get_mm_counter(mm, rss) << (PAGE_SHIFT-10), data << (PAGE_SHIFT-10), mm->stack_vm << (PAGE_SHIFT-10), text, lib, (PTRS_PER_PTE*sizeof(pte_t)*mm->nr_ptes) >> 10); @@ -39,11 +39,13 @@ unsigned long task_vsize(struct mm_struc int task_statm(struct mm_struct *mm, int *shared, int *text, int *data, int *resident) { - *shared = mm->rss - mm->anon_rss; + int rss = get_mm_counter(mm, rss); + + *shared = rss - get_mm_counter(mm, anon_rss); *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT; *data = mm->total_vm - mm->shared_vm; - *resident = mm->rss; + *resident = rss; return mm->total_vm; } diff -puN include/asm-arm26/tlb.h~mm-counter-operations-through-macros include/asm-arm26/tlb.h --- 25/include/asm-arm26/tlb.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/asm-arm26/tlb.h 2005-03-14 21:38:16.000000000 -0800 @@ -37,11 +37,11 @@ tlb_finish_mmu(struct mmu_gather *tlb, u { struct mm_struct *mm = tlb->mm; unsigned long freed = tlb->freed; - int rss = mm->rss; + int rss = get_mm_counter(mm, rss); if (rss < freed) freed = rss; - mm->rss = rss - freed; + add_mm_counter(mm, rss, -freed); if (freed) { flush_tlb_mm(mm); diff -puN include/asm-arm/tlb.h~mm-counter-operations-through-macros include/asm-arm/tlb.h --- 25/include/asm-arm/tlb.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/asm-arm/tlb.h 2005-03-14 21:38:16.000000000 -0800 @@ -54,11 +54,11 @@ tlb_finish_mmu(struct mmu_gather *tlb, u { struct mm_struct *mm = tlb->mm; unsigned long freed = tlb->freed; - int rss = mm->rss; + int rss = get_mm_counter(mm, rss); if (rss < freed) freed = rss; - mm->rss = rss - freed; + add_mm_counter(mm, rss, -freed); if (freed) { flush_tlb_mm(mm); diff -puN include/asm-generic/tlb.h~mm-counter-operations-through-macros include/asm-generic/tlb.h --- 25/include/asm-generic/tlb.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/asm-generic/tlb.h 2005-03-14 21:38:16.000000000 -0800 @@ -88,11 +88,11 @@ tlb_finish_mmu(struct mmu_gather *tlb, u { int freed = tlb->freed; struct mm_struct *mm = tlb->mm; - int rss = mm->rss; + int rss = get_mm_counter(mm, rss); if (rss < freed) freed = rss; - mm->rss = rss - freed; + add_mm_counter(mm, rss, -freed); tlb_flush_mmu(tlb, start, end); /* keep the page table cache within bounds */ diff -puN include/asm-ia64/tlb.h~mm-counter-operations-through-macros include/asm-ia64/tlb.h --- 25/include/asm-ia64/tlb.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/asm-ia64/tlb.h 2005-03-14 21:38:16.000000000 -0800 @@ -161,11 +161,11 @@ tlb_finish_mmu (struct mmu_gather *tlb, { unsigned long freed = tlb->freed; struct mm_struct *mm = tlb->mm; - unsigned long rss = mm->rss; + unsigned long rss = get_mm_counter(mm, rss); if (rss < freed) freed = rss; - mm->rss = rss - freed; + add_mm_counter(mm, rss, -freed); /* * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and * tlb->end_addr. diff -puN include/asm-sparc64/tlb.h~mm-counter-operations-through-macros include/asm-sparc64/tlb.h --- 25/include/asm-sparc64/tlb.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/asm-sparc64/tlb.h 2005-03-14 21:38:16.000000000 -0800 @@ -80,11 +80,11 @@ static inline void tlb_finish_mmu(struct { unsigned long freed = mp->freed; struct mm_struct *mm = mp->mm; - unsigned long rss = mm->rss; + unsigned long rss = get_mm_counter(mm, rss); if (rss < freed) freed = rss; - mm->rss = rss - freed; + add_mm_counter(mm, rss, -freed); tlb_flush_mmu(mp); diff -puN include/linux/sched.h~mm-counter-operations-through-macros include/linux/sched.h --- 25/include/linux/sched.h~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/include/linux/sched.h 2005-03-14 21:43:28.000000000 -0800 @@ -204,6 +204,13 @@ arch_get_unmapped_area_topdown(struct fi extern void arch_unmap_area(struct vm_area_struct *area); extern void arch_unmap_area_topdown(struct vm_area_struct *area); +#define set_mm_counter(mm, member, value) (mm)->_##member = (value) +#define get_mm_counter(mm, member) ((mm)->_##member) +#define add_mm_counter(mm, member, value) (mm)->_##member += (value) +#define inc_mm_counter(mm, member) (mm)->_##member++ +#define dec_mm_counter(mm, member) (mm)->_##member-- +typedef unsigned long mm_counter_t; +#define MM_COUNTER_T(member) mm_counter_t _##member struct mm_struct { struct vm_area_struct * mmap; /* list of VMAs */ @@ -220,7 +227,7 @@ struct mm_struct { atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ int map_count; /* number of VMAs */ struct rw_semaphore mmap_sem; - spinlock_t page_table_lock; /* Protects page tables, mm->rss, mm->anon_rss */ + spinlock_t page_table_lock; /* Protects page tables and some counters */ struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung * together off init_mm.mmlist, and are protected @@ -230,9 +237,13 @@ struct mm_struct { unsigned long start_code, end_code, start_data, end_data; unsigned long start_brk, brk, start_stack; unsigned long arg_start, arg_end, env_start, env_end; - unsigned long rss, anon_rss, total_vm, locked_vm, shared_vm; + unsigned long total_vm, locked_vm, shared_vm; unsigned long exec_vm, stack_vm, reserved_vm, def_flags, nr_ptes; + /* Special counters protected by the page_table_lock */ + MM_COUNTER_T(rss); + MM_COUNTER_T(anon_rss); + unsigned long saved_auxv[42]; /* for /proc/PID/auxv */ unsigned dumpable:1; diff -puN kernel/acct.c~mm-counter-operations-through-macros kernel/acct.c --- 25/kernel/acct.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/kernel/acct.c 2005-03-14 21:38:16.000000000 -0800 @@ -542,7 +542,7 @@ void acct_update_integrals(struct task_s if (delta == 0) return; tsk->acct_stimexpd = tsk->stime; - tsk->acct_rss_mem1 += delta * tsk->mm->rss; + tsk->acct_rss_mem1 += delta * get_mm_counter(tsk->mm, rss); tsk->acct_vm_mem1 += delta * tsk->mm->total_vm; } } diff -puN kernel/fork.c~mm-counter-operations-through-macros kernel/fork.c --- 25/kernel/fork.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/kernel/fork.c 2005-03-14 21:38:16.000000000 -0800 @@ -195,8 +195,8 @@ static inline int dup_mmap(struct mm_str mm->mmap_cache = NULL; mm->free_area_cache = oldmm->mmap_base; mm->map_count = 0; - mm->rss = 0; - mm->anon_rss = 0; + set_mm_counter(mm, rss, 0); + set_mm_counter(mm, anon_rss, 0); cpus_clear(mm->cpu_vm_mask); mm->mm_rb = RB_ROOT; rb_link = &mm->mm_rb.rb_node; @@ -492,7 +492,7 @@ static int copy_mm(unsigned long clone_f if (retval) goto free_pt; - mm->hiwater_rss = mm->rss; + mm->hiwater_rss = get_mm_counter(mm,rss); mm->hiwater_vm = mm->total_vm; good_mm: diff -puN mm/fremap.c~mm-counter-operations-through-macros mm/fremap.c --- 25/mm/fremap.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/fremap.c 2005-03-14 21:38:16.000000000 -0800 @@ -39,7 +39,7 @@ static inline void zap_pte(struct mm_str set_page_dirty(page); page_remove_rmap(page); page_cache_release(page); - mm->rss--; + dec_mm_counter(mm, rss); } } } else { @@ -92,7 +92,7 @@ int install_page(struct mm_struct *mm, s zap_pte(mm, vma, addr, pte); - mm->rss++; + inc_mm_counter(mm,rss); flush_icache_page(vma, page); set_pte_at(mm, addr, pte, mk_pte(page, prot)); page_add_file_rmap(page); diff -puN mm/memory.c~mm-counter-operations-through-macros mm/memory.c --- 25/mm/memory.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/memory.c 2005-03-14 21:38:58.000000000 -0800 @@ -309,9 +309,9 @@ copy_one_pte(struct mm_struct *dst_mm, s pte = pte_mkclean(pte); pte = pte_mkold(pte); get_page(page); - dst_mm->rss++; + inc_mm_counter(dst_mm, rss); if (PageAnon(page)) - dst_mm->anon_rss++; + inc_mm_counter(dst_mm, anon_rss); set_pte_at(dst_mm, addr, dst_pte, pte); page_dup_rmap(page); } @@ -475,7 +475,7 @@ static void zap_pte_range(struct mmu_gat if (pte_dirty(ptent)) set_page_dirty(page); if (PageAnon(page)) - tlb->mm->anon_rss--; + dec_mm_counter(tlb->mm, anon_rss); else if (pte_young(ptent)) mark_page_accessed(page); tlb->freed++; @@ -1219,9 +1219,9 @@ static int do_wp_page(struct mm_struct * page_table = pte_offset_map(pmd, address); if (likely(pte_same(*page_table, pte))) { if (PageAnon(old_page)) - mm->anon_rss--; + dec_mm_counter(mm, anon_rss); if (PageReserved(old_page)) - ++mm->rss; + inc_mm_counter(mm, rss); else page_remove_rmap(old_page); flush_cache_page(vma, address, pfn); @@ -1627,7 +1627,7 @@ static int do_swap_page(struct mm_struct if (vm_swap_full()) remove_exclusive_swap_page(page); - mm->rss++; + inc_mm_counter(mm, rss); pte = mk_pte(page, vma->vm_page_prot); if (write_access && can_share_swap_page(page)) { pte = maybe_mkwrite(pte_mkdirty(pte), vma); @@ -1691,7 +1691,7 @@ do_anonymous_page(struct mm_struct *mm, spin_unlock(&mm->page_table_lock); goto out; } - mm->rss++; + inc_mm_counter(mm, rss); entry = maybe_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)), vma); @@ -1807,7 +1807,7 @@ retry: /* Only go through if we didn't race with anybody else... */ if (pte_none(*page_table)) { if (!PageReserved(new_page)) - ++mm->rss; + inc_mm_counter(mm, rss); flush_icache_page(vma, new_page); entry = mk_pte(new_page, vma->vm_page_prot); @@ -2112,8 +2112,10 @@ EXPORT_SYMBOL(vmalloc_to_pfn); void update_mem_hiwater(struct task_struct *tsk) { if (tsk->mm) { - if (tsk->mm->hiwater_rss < tsk->mm->rss) - tsk->mm->hiwater_rss = tsk->mm->rss; + unsigned long rss = get_mm_counter(tsk->mm, rss); + + if (tsk->mm->hiwater_rss < rss) + tsk->mm->hiwater_rss = rss; if (tsk->mm->hiwater_vm < tsk->mm->total_vm) tsk->mm->hiwater_vm = tsk->mm->total_vm; } diff -puN mm/mmap.c~mm-counter-operations-through-macros mm/mmap.c --- 25/mm/mmap.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/mmap.c 2005-03-14 21:38:16.000000000 -0800 @@ -1978,7 +1978,7 @@ void exit_mmap(struct mm_struct *mm) vma = mm->mmap; mm->mmap = mm->mmap_cache = NULL; mm->mm_rb = RB_ROOT; - mm->rss = 0; + set_mm_counter(mm, rss, 0); mm->total_vm = 0; mm->locked_vm = 0; diff -puN mm/nommu.c~mm-counter-operations-through-macros mm/nommu.c --- 25/mm/nommu.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/nommu.c 2005-03-14 21:38:16.000000000 -0800 @@ -961,9 +961,11 @@ void arch_unmap_area(struct vm_area_stru void update_mem_hiwater(struct task_struct *tsk) { + unsigned long rss = get_mm_counter(tsk->mm, rss); + if (likely(tsk->mm)) { - if (tsk->mm->hiwater_rss < tsk->mm->rss) - tsk->mm->hiwater_rss = tsk->mm->rss; + if (tsk->mm->hiwater_rss < rss) + tsk->mm->hiwater_rss = rss; if (tsk->mm->hiwater_vm < tsk->mm->total_vm) tsk->mm->hiwater_vm = tsk->mm->total_vm; } diff -puN mm/rmap.c~mm-counter-operations-through-macros mm/rmap.c --- 25/mm/rmap.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/rmap.c 2005-03-14 21:38:16.000000000 -0800 @@ -257,7 +257,7 @@ static int page_referenced_one(struct pa pte_t *pte; int referenced = 0; - if (!mm->rss) + if (!get_mm_counter(mm, rss)) goto out; address = vma_address(page, vma); if (address == -EFAULT) @@ -436,7 +436,7 @@ void page_add_anon_rmap(struct page *pag BUG_ON(PageReserved(page)); BUG_ON(!anon_vma); - vma->vm_mm->anon_rss++; + inc_mm_counter(vma->vm_mm, anon_rss); anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; index = (address - vma->vm_start) >> PAGE_SHIFT; @@ -509,7 +509,7 @@ static int try_to_unmap_one(struct page pte_t pteval; int ret = SWAP_AGAIN; - if (!mm->rss) + if (!get_mm_counter(mm, rss)) goto out; address = vma_address(page, vma); if (address == -EFAULT) @@ -595,10 +595,10 @@ static int try_to_unmap_one(struct page } set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); BUG_ON(pte_file(*pte)); - mm->anon_rss--; + dec_mm_counter(mm, anon_rss); } - mm->rss--; + inc_mm_counter(mm, rss); page_remove_rmap(page); page_cache_release(page); @@ -703,7 +703,7 @@ static void try_to_unmap_cluster(unsigne page_remove_rmap(page); page_cache_release(page); - mm->rss--; + dec_mm_counter(mm, rss); (*mapcount)--; } @@ -802,7 +802,7 @@ static int try_to_unmap_file(struct page if (vma->vm_flags & (VM_LOCKED|VM_RESERVED)) continue; cursor = (unsigned long) vma->vm_private_data; - while (vma->vm_mm->rss && + while (get_mm_counter(vma->vm_mm, rss) && cursor < max_nl_cursor && cursor < vma->vm_end - vma->vm_start) { try_to_unmap_cluster(cursor, &mapcount, vma); diff -puN mm/swapfile.c~mm-counter-operations-through-macros mm/swapfile.c --- 25/mm/swapfile.c~mm-counter-operations-through-macros 2005-03-14 21:38:16.000000000 -0800 +++ 25-akpm/mm/swapfile.c 2005-03-14 21:38:16.000000000 -0800 @@ -404,7 +404,7 @@ void free_swap_and_cache(swp_entry_t ent static void unuse_pte(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, swp_entry_t entry, struct page *page) { - vma->vm_mm->rss++; + inc_mm_counter(vma->vm_mm, rss); get_page(page); set_pte_at(vma->vm_mm, addr, pte, pte_mkold(mk_pte(page, vma->vm_page_prot))); _