From: Christoph Lameter Changelog * Provide atomic pte operations for ia64 * Enhanced parallelism in page fault handler if applied together with the generic patch Signed-off-by: Christoph Lameter Signed-off-by: Andrew Morton --- 25-akpm/include/asm-ia64/pgalloc.h | 18 ++++++++++++++++++ 25-akpm/include/asm-ia64/pgtable.h | 29 ++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 7 deletions(-) diff -puN include/asm-ia64/pgalloc.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations include/asm-ia64/pgalloc.h --- 25/include/asm-ia64/pgalloc.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations Thu Dec 2 12:39:13 2004 +++ 25-akpm/include/asm-ia64/pgalloc.h Thu Dec 2 12:39:13 2004 @@ -34,6 +34,10 @@ #define pmd_quicklist (local_cpu_data->pmd_quick) #define pgtable_cache_size (local_cpu_data->pgtable_cache_sz) +/* Empty entries of PMD and PGD */ +#define PMD_NONE 0 +#define PGD_NONE 0 + static inline pgd_t* pgd_alloc_one_fast (struct mm_struct *mm) { @@ -64,12 +68,19 @@ pgd_free (pgd_t *pgd) preempt_enable(); } + static inline void pgd_populate (struct mm_struct *mm, pgd_t *pgd_entry, pmd_t *pmd) { pgd_val(*pgd_entry) = __pa(pmd); } +/* Atomic populate */ +static inline int +pgd_test_and_populate (struct mm_struct *mm, pgd_t *pgd_entry, pmd_t *pmd) +{ + return ia64_cmpxchg8_acq(pgd_entry,__pa(pmd), PGD_NONE) == PGD_NONE; +} static inline pmd_t* pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr) @@ -118,6 +129,13 @@ pmd_populate (struct mm_struct *mm, pmd_ pmd_val(*pmd_entry) = page_to_phys(pte); } +/* Atomic populate */ +static inline int +pmd_test_and_populate (struct mm_struct *mm, pmd_t *pmd_entry, struct page *pte) +{ + return ia64_cmpxchg8_acq(pmd_entry, page_to_phys(pte), PMD_NONE) == PMD_NONE; +} + static inline void pmd_populate_kernel (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte) { diff -puN include/asm-ia64/pgtable.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations include/asm-ia64/pgtable.h --- 25/include/asm-ia64/pgtable.h~page-fault-scalability-patch-v11-ia64-atomic-pte-operations Thu Dec 2 12:39:13 2004 +++ 25-akpm/include/asm-ia64/pgtable.h Thu Dec 2 12:39:13 2004 @@ -407,6 +407,26 @@ ptep_mkdirty (pte_t *ptep) #endif } +/* + * IA-64 doesn't have any external MMU info: the page tables contain all the necessary + * information. However, we use this routine to take care of any (delayed) i-cache + * flushing that may be necessary. + */ +extern void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte); + +static inline int +ptep_cmpxchg (struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t oldval, pte_t newval) +{ + /* + * IA64 defers icache flushes. If the new pte is executable we may + * have to flush the icache to insure cache coherency immediately + * after the cmpxchg. + */ + if (pte_exec(newval)) + update_mmu_cache(vma, addr, newval); + return ia64_cmpxchg8_acq(&ptep->pte, newval.pte, oldval.pte) == oldval.pte; +} + static inline int pte_same (pte_t a, pte_t b) { @@ -469,13 +489,6 @@ extern void hugetlb_free_pgtables(struct struct vm_area_struct * prev, unsigned long start, unsigned long end); #endif -/* - * IA-64 doesn't have any external MMU info: the page tables contain all the necessary - * information. However, we use this routine to take care of any (delayed) i-cache - * flushing that may be necessary. - */ -extern void update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte); - #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* * Update PTEP with ENTRY, which is guaranteed to be a less @@ -553,6 +566,8 @@ do { \ #define __HAVE_ARCH_PTEP_MKDIRTY #define __HAVE_ARCH_PTE_SAME #define __HAVE_ARCH_PGD_OFFSET_GATE +#define __HAVE_ARCH_ATOMIC_TABLE_OPS +#define __HAVE_ARCH_LOCK_TABLE_OPS #include #include _