diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/arch/i386/Kconfig 802-banana_split/arch/i386/Kconfig --- 801-separate_pmd/arch/i386/Kconfig Wed Aug 13 20:48:59 2003 +++ 802-banana_split/arch/i386/Kconfig Wed Aug 13 20:51:03 2003 @@ -706,7 +706,6 @@ choice config 05GB bool "3.5 GB" - depends on !HIGHMEM64G config 1GB bool "3 GB" diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/arch/i386/mm/init.c 802-banana_split/arch/i386/mm/init.c --- 801-separate_pmd/arch/i386/mm/init.c Wed Aug 13 20:50:53 2003 +++ 802-banana_split/arch/i386/mm/init.c Wed Aug 13 20:51:03 2003 @@ -121,6 +121,24 @@ static void __init page_table_range_init } } + +/* + * Abstract out using large pages when mapping KVA, or the SMP identity + * mapping + */ +void pmd_map_pfn_range(pmd_t* pmd_entry, unsigned long pfn, unsigned long max_pfn) +{ + int pte_ofs; + /* Map with big pages if possible, otherwise create normal page tables. */ + if (cpu_has_pse) { + set_pmd(pmd_entry, pfn_pmd(pfn, PAGE_KERNEL_LARGE)); + pfn += PTRS_PER_PTE; + } else { + pte_t* pte = one_page_table_init(pmd_entry); + for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE && pfn < max_pfn; pte++, pfn++, pte_ofs++) + set_pte(pte, pfn_pte(pfn, PAGE_KERNEL)); + } +} /* * This maps the physical memory to kernel virtual address space, a total * of max_low_pfn pages, by creating page tables starting from address @@ -131,8 +149,7 @@ static void __init kernel_physical_mappi unsigned long pfn; pgd_t *pgd; pmd_t *pmd; - pte_t *pte; - int pgd_idx, pmd_idx, pte_ofs; + int pgd_idx, pmd_idx; pgd_idx = pgd_index(PAGE_OFFSET); pgd = pgd_base + pgd_idx; @@ -142,21 +159,48 @@ static void __init kernel_physical_mappi pmd = one_md_table_init(pgd); if (pfn >= max_low_pfn) continue; - for (pmd_idx = 0; pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_idx++) { - /* Map with big pages if possible, otherwise create normal page tables. */ - if (cpu_has_pse) { - set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE)); - pfn += PTRS_PER_PTE; - } else { - pte = one_page_table_init(pmd); - - for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn; pte++, pfn++, pte_ofs++) - set_pte(pte, pfn_pte(pfn, PAGE_KERNEL)); - } + + /* beware of starting KVA in the middle of a pmd. */ + if( pgd_idx == pgd_index(PAGE_OFFSET) ) { + pmd_idx = pmd_index(PAGE_OFFSET); + pmd = &pmd[pmd_idx]; + } else + pmd_idx = 0; + + for (; pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_idx++) { + pmd_map_pfn_range(pmd, pfn, max_low_pfn); + pfn += PTRS_PER_PTE; } } } +/* + * Add low memory identity-mappings - SMP needs it when + * starting up on an AP from real-mode. In the non-PAE + * case we already have these mappings through head.S. + * All user-space mappings are explicitly cleared after + * SMP startup in zap_low_mappings(). + */ +static void __init low_physical_mapping_init(pgd_t *pgd_base) +{ +#if CONFIG_X86_PAE + unsigned long pfn = 0; + int pmd_ofs = 0; + pmd_t *pmd = one_md_table_init(pgd_base); + + if(!cpu_has_pse) { + printk("PAE enabled, but no support for PSE (large pages)!\n"); + printk("this is likely to waste some RAM."); + } + + for (; pmd_ofs < PTRS_PER_PMD && pfn <= max_low_pfn; pmd++, pmd_ofs++) { + pmd_map_pfn_range(pmd, pfn, max_low_pfn); + pfn += PTRS_PER_PTE; + } +#endif +} + + static inline int page_kills_ppro(unsigned long pagenr) { if (pagenr >= 0x70000 && pagenr <= 0x7003F) @@ -217,7 +261,7 @@ void __init permanent_kmaps_init(pgd_t * pgd = swapper_pg_dir + pgd_index(vaddr); pmd = pmd_offset(pgd, vaddr); pte = pte_offset_kernel(pmd, vaddr); - pkmap_page_table = pte; + pkmap_page_table = pte; } void __init one_highpage_init(struct page *page, int pfn, int bad_ppro) @@ -282,6 +326,7 @@ static void __init pagetable_init (void) } kernel_physical_mapping_init(pgd_base); + low_physical_mapping_init(pgd_base); remap_numa_kva(); /* @@ -290,19 +335,7 @@ static void __init pagetable_init (void) */ vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK; page_table_range_init(vaddr, 0, pgd_base); - permanent_kmaps_init(pgd_base); - -#ifdef CONFIG_X86_PAE - /* - * Add low memory identity-mappings - SMP needs it when - * starting up on an AP from real-mode. In the non-PAE - * case we already have these mappings through head.S. - * All user-space mappings are explicitly cleared after - * SMP startup. - */ - pgd_base[0] = pgd_base[USER_PTRS_PER_PGD]; -#endif } void zap_low_mappings (void) @@ -314,7 +347,7 @@ void zap_low_mappings (void) * Note that "pgd_clear()" doesn't do it for * us, because pgd_clear() is a no-op on i386. */ - for (i = 0; i < USER_PTRS_PER_PGD; i++) + for (i = 0; i < FIRST_KERNEL_PGD_PTR; i++) #ifdef CONFIG_X86_PAE set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page))); #else diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/arch/i386/mm/pgtable.c 802-banana_split/arch/i386/mm/pgtable.c --- 801-separate_pmd/arch/i386/mm/pgtable.c Wed Aug 13 20:50:53 2003 +++ 802-banana_split/arch/i386/mm/pgtable.c Wed Aug 13 20:51:03 2003 @@ -159,10 +159,23 @@ void pmd_ctor(void *pmd, kmem_cache_t *c void kernel_pmd_ctor(void *__pmd, kmem_cache_t *kernel_pmd_cache, unsigned long flags) { + pmd_t *pmd = __pmd; int i; - for (i=USER_PGD_PTRS; i __USER_PTRS_PER_PGD*PGDIR_SIZE ? 1 : 0) +#define PARTIAL_PMD ((TASK_SIZE % PGDIR_SIZE)/PMD_SIZE) +#define USER_PTRS_PER_PGD (PARTIAL_PGD + __USER_PTRS_PER_PGD) +#ifndef __ASSEMBLY__ +static inline int USER_PTRS_PER_PMD(int pgd_index) { + if (pgd_index < __USER_PTRS_PER_PGD) + return PTRS_PER_PMD; + else if (PARTIAL_PMD && (pgd_index == __USER_PTRS_PER_PGD)) + return (PTRS_PER_PMD-PARTIAL_PMD); + else + return 0; +} +#endif + #define FIRST_USER_PGD_NR 0 #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-ia64/pgtable.h 802-banana_split/include/asm-ia64/pgtable.h --- 801-separate_pmd/include/asm-ia64/pgtable.h Tue Jun 24 21:29:24 2003 +++ 802-banana_split/include/asm-ia64/pgtable.h Wed Aug 13 20:51:03 2003 @@ -92,6 +92,7 @@ #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (__IA64_UL(1) << (PAGE_SHIFT-3)) #define USER_PTRS_PER_PGD (5*PTRS_PER_PGD/8) /* regions 0-4 are user regions */ +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 /* diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-m68k/pgtable.h 802-banana_split/include/asm-m68k/pgtable.h --- 801-separate_pmd/include/asm-m68k/pgtable.h Sat Jun 14 18:37:35 2003 +++ 802-banana_split/include/asm-m68k/pgtable.h Wed Aug 13 20:51:03 2003 @@ -58,6 +58,7 @@ #define PTRS_PER_PGD 128 #endif #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 /* Virtual address region for use by kernel_map() */ diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-parisc/pgtable.h 802-banana_split/include/asm-parisc/pgtable.h --- 801-separate_pmd/include/asm-parisc/pgtable.h Tue Aug 5 20:01:43 2003 +++ 802-banana_split/include/asm-parisc/pgtable.h Wed Aug 13 20:51:19 2003 @@ -81,6 +81,7 @@ #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1UL << (PAGE_SHIFT - PT_NLEVELS)) #define USER_PTRS_PER_PGD PTRS_PER_PGD +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) /* Definitions for 2nd level */ #define pgtable_cache_init() do { } while (0) diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-ppc/pgtable.h 802-banana_split/include/asm-ppc/pgtable.h --- 801-separate_pmd/include/asm-ppc/pgtable.h Sat Jun 14 18:37:36 2003 +++ 802-banana_split/include/asm-ppc/pgtable.h Wed Aug 13 20:51:19 2003 @@ -83,6 +83,7 @@ extern unsigned long ioremap_bot, iorema #define PTRS_PER_PMD 1 #define PTRS_PER_PGD 1024 #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-ppc64/pgtable.h 802-banana_split/include/asm-ppc64/pgtable.h --- 801-separate_pmd/include/asm-ppc64/pgtable.h Sat Jun 14 18:37:36 2003 +++ 802-banana_split/include/asm-ppc64/pgtable.h Wed Aug 13 20:51:19 2003 @@ -36,6 +36,7 @@ #define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) #define USER_PTRS_PER_PGD (1024) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \ diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-sh/pgtable.h 802-banana_split/include/asm-sh/pgtable.h --- 801-separate_pmd/include/asm-sh/pgtable.h Wed Jul 2 21:59:15 2003 +++ 802-banana_split/include/asm-sh/pgtable.h Wed Aug 13 20:51:19 2003 @@ -41,6 +41,7 @@ extern unsigned long empty_zero_page[102 #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define PTE_PHYS_MASK 0x1ffff000 diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-sparc/pgtable.h 802-banana_split/include/asm-sparc/pgtable.h --- 801-separate_pmd/include/asm-sparc/pgtable.h Sat May 10 18:35:03 2003 +++ 802-banana_split/include/asm-sparc/pgtable.h Wed Aug 13 20:51:19 2003 @@ -125,6 +125,7 @@ BTFIXUPDEF_INT(page_kernel) #define PTRS_PER_PMD BTFIXUP_SIMM13(ptrs_per_pmd) #define PTRS_PER_PGD BTFIXUP_SIMM13(ptrs_per_pgd) #define USER_PTRS_PER_PGD BTFIXUP_SIMM13(user_ptrs_per_pgd) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define PAGE_NONE __pgprot(BTFIXUP_INT(page_none)) diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-sparc64/pgtable.h 802-banana_split/include/asm-sparc64/pgtable.h --- 801-separate_pmd/include/asm-sparc64/pgtable.h Wed Mar 26 22:54:37 2003 +++ 802-banana_split/include/asm-sparc64/pgtable.h Wed Aug 13 20:51:19 2003 @@ -93,6 +93,7 @@ /* Kernel has a separate 44bit address space. */ #define USER_PTRS_PER_PGD ((const int)(test_thread_flag(TIF_32BIT)) ? \ (1) : (PTRS_PER_PGD)) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define pte_ERROR(e) __builtin_trap() diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-um/pgtable.h 802-banana_split/include/asm-um/pgtable.h --- 801-separate_pmd/include/asm-um/pgtable.h Fri May 30 19:02:21 2003 +++ 802-banana_split/include/asm-um/pgtable.h Wed Aug 13 20:51:19 2003 @@ -40,6 +40,7 @@ extern unsigned long *empty_zero_page; #define PTRS_PER_PMD 1 #define PTRS_PER_PGD 1024 #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define pte_ERROR(e) \ diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/include/asm-x86_64/pgtable.h 802-banana_split/include/asm-x86_64/pgtable.h --- 801-separate_pmd/include/asm-x86_64/pgtable.h Wed Jul 2 21:59:15 2003 +++ 802-banana_split/include/asm-x86_64/pgtable.h Wed Aug 13 20:51:19 2003 @@ -112,6 +112,7 @@ static inline void set_pml4(pml4_t *dst, #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) +#define USER_PTRS_PER_PMD(x) (PTRS_PER_PMD) #define FIRST_USER_PGD_NR 0 #define USER_PGD_PTRS (PAGE_OFFSET >> PGDIR_SHIFT) diff -urpN -X /home/fletch/.diff.exclude 801-separate_pmd/mm/memory.c 802-banana_split/mm/memory.c --- 801-separate_pmd/mm/memory.c Wed Aug 13 20:29:24 2003 +++ 802-banana_split/mm/memory.c Wed Aug 13 20:51:19 2003 @@ -100,9 +100,10 @@ static inline void free_one_pmd(struct m pte_free_tlb(tlb, page); } -static inline void free_one_pgd(struct mmu_gather *tlb, pgd_t * dir) +static inline void free_one_pgd(struct mmu_gather *tlb, pgd_t * pgd, unsigned long pgdi) { pmd_t * pmd, * md, * emd; + pgd_t *dir = pgd + pgdi; if (pgd_none(*dir)) return; @@ -126,7 +127,7 @@ static inline void free_one_pgd(struct m * found at 0x80000000 onwards. The loop below compiles instead * to be terminated by unsigned address comparison using "jb". */ - for (md = pmd, emd = pmd + PTRS_PER_PMD; md < emd; md++) + for (md = pmd, emd = pmd + USER_PTRS_PER_PMD(pgdi); md < emd; md++) free_one_pmd(tlb,md); pmd_free_tlb(tlb, pmd); } @@ -140,11 +141,11 @@ static inline void free_one_pgd(struct m void clear_page_tables(struct mmu_gather *tlb, unsigned long first, int nr) { pgd_t * page_dir = tlb->mm->pgd; - - page_dir += first; + int index = first; + do { - free_one_pgd(tlb, page_dir); - page_dir++; + free_one_pgd(tlb, page_dir, index); + index++; } while (--nr); }