diff -urN x86-64-ref/arch/x86_64/kernel/setup.c x86-64/arch/x86_64/kernel/setup.c --- x86-64-ref/arch/x86_64/kernel/setup.c Tue Mar 12 10:42:25 2002 +++ x86-64/arch/x86_64/kernel/setup.c Tue Mar 12 09:04:16 2002 @@ -53,7 +53,12 @@ extern void mcheck_init(struct cpuinfo_x86 *c); char ignore_irq13; /* set if exception 16 works */ -struct cpuinfo_x86 boot_cpu_data = { 0, 0, 0, 0, -1, 1, 0, 0, -1 }; +struct cpuinfo_x86 boot_cpu_data = { + wp_works_ok: -1, + hlt_works_ok: 1, + cpuid_level: -1, + pte_quick: LIST_HEAD_INIT(boot_cpu_data.pte_quick), +}; unsigned long mmu_cr4_features; diff -urN x86-64-ref/arch/x86_64/kernel/smpboot.c x86-64/arch/x86_64/kernel/smpboot.c --- x86-64-ref/arch/x86_64/kernel/smpboot.c Tue Mar 12 10:42:25 2002 +++ x86-64/arch/x86_64/kernel/smpboot.c Tue Mar 12 09:34:42 2002 @@ -149,6 +149,10 @@ struct cpuinfo_x86 *c = cpu_data + id; *c = boot_cpu_data; + INIT_LIST_HEAD(&c->pte_quick); + c->pmd_quick = 0; + c->pgd_quick = 0; + c->pgtable_cache_sz = 0; identify_cpu(c); /* * Mask B, Pentium, but not Pentium MMX diff -urN x86-64-ref/arch/x86_64/mm/init.c x86-64/arch/x86_64/mm/init.c --- x86-64-ref/arch/x86_64/mm/init.c Tue Mar 12 10:42:25 2002 +++ x86-64/arch/x86_64/mm/init.c Tue Mar 12 10:08:19 2002 @@ -42,21 +42,21 @@ int do_check_pgt_cache(int low, int high) { int freed = 0; - if(read_pda(pgtable_cache_sz) > high) { + if(pgtable_cache_size > high) { do { - if (read_pda(pgd_quick)) { - pgd_free_slow(pgd_alloc_one_fast()); + if (pgd_quicklist) { + free_pgd_slow(pgd_alloc(NULL)); freed++; } - if (read_pda(pmd_quick)) { + if (pmd_quicklist) { pmd_free_slow(pmd_alloc_one_fast(NULL, 0)); freed++; } - if (read_pda(pte_quick)) { - pte_free_slow(pte_alloc_one_fast(NULL, 0)); + if (!list_empty(&pte_quicklist)) { + pte_free_slow(pte_alloc_one_fast_lifo(NULL, 0)); freed++; } - } while(read_pda(pgtable_cache_sz) > low); + } while(pgtable_cache_size > low); } return freed; } @@ -89,7 +89,7 @@ printk("%d reserved pages\n",reserved); printk("%d pages shared\n",shared); printk("%d pages swap cached\n",cached); - printk("%ld pages in page table cache\n",read_pda(pgtable_cache_sz)); + printk("%ld pages in page table cache\n", pgtable_cache_size); show_buffers(); } diff -urN x86-64-ref/arch/x86_64/tools/offset.c x86-64/arch/x86_64/tools/offset.c --- x86-64-ref/arch/x86_64/tools/offset.c Tue Mar 12 10:42:25 2002 +++ x86-64/arch/x86_64/tools/offset.c Tue Mar 12 09:32:41 2002 @@ -40,10 +40,6 @@ ENTRY(pcurrent); ENTRY(irqrsp); ENTRY(irqcount); - ENTRY(pgd_quick); - ENTRY(pmd_quick); - ENTRY(pte_quick); - ENTRY(pgtable_cache_sz); ENTRY(cpunumber); ENTRY(irqstackptr); ENTRY(level4_pgt); diff -urN x86-64-ref/include/asm-x86_64/pda.h x86-64/include/asm-x86_64/pda.h --- x86-64-ref/include/asm-x86_64/pda.h Tue Mar 12 10:42:25 2002 +++ x86-64/include/asm-x86_64/pda.h Tue Mar 12 09:02:20 2002 @@ -16,11 +16,6 @@ struct task_struct *pcurrent; /* Current process */ int irqcount; /* Irq nesting counter. Starts with -1 */ int cpunumber; /* Logical CPU number */ - /* XXX: could be a single list */ - unsigned long *pgd_quick; - unsigned long *pmd_quick; - unsigned long *pte_quick; - unsigned long pgtable_cache_sz; char *irqstackptr; /* top of irqstack */ unsigned long volatile *level4_pgt; } ____cacheline_aligned; diff -urN x86-64-ref/include/asm-x86_64/pgalloc.h x86-64/include/asm-x86_64/pgalloc.h --- x86-64-ref/include/asm-x86_64/pgalloc.h Tue Mar 12 10:42:25 2002 +++ x86-64/include/asm-x86_64/pgalloc.h Tue Mar 12 10:02:26 2002 @@ -4,15 +4,15 @@ #include #include #include -#include #include -#include +#include -#define inc_pgcache_size() add_pda(pgtable_cache_sz,1UL) -#define dec_pgcache_size() sub_pda(pgtable_cache_sz,1UL) +#define pgd_quicklist (current_cpu_data.pgd_quick) +#define pmd_quicklist (current_cpu_data.pmd_quick) +#define pte_quicklist (current_cpu_data.pte_quick) +#define pgtable_cache_size (current_cpu_data.pgtable_cache_sz) -#define pmd_populate(mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE | __pa(pte))) +#define pmd_populate(mm, pmd, page) set_pmd(pmd, mk_pmd(page, __pgprot(_PAGE_TABLE))) #define pgd_populate(mm, pgd, pmd) \ set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(pmd))) @@ -25,41 +25,30 @@ return ret; } -extern __inline__ pmd_t *get_pmd_fast(void) -{ - unsigned long *ret; - - if ((ret = read_pda(pmd_quick)) != NULL) { - write_pda(pmd_quick, (unsigned long *)(*ret)); - ret[0] = 0; - dec_pgcache_size(); - } else - ret = (unsigned long *)get_pmd_slow(); - return (pmd_t *)ret; -} - extern __inline__ void pmd_free(pmd_t *pmd) { - *(unsigned long *)pmd = (unsigned long) read_pda(pmd_quick); - write_pda(pmd_quick,(unsigned long *) pmd); - inc_pgcache_size(); + *(unsigned long *)pmd = (unsigned long) pmd_quicklist; + pmd_quicklist = (unsigned long *) pmd; + pgtable_cache_size++; } extern __inline__ void pmd_free_slow(pmd_t *pmd) { - if ((unsigned long)pmd & (PAGE_SIZE-1)) - BUG(); +#ifdef CONFIG_CHECKING + if ((unsigned long)pmd & (PAGE_SIZE-1)) + BUG(); +#endif free_page((unsigned long)pmd); } static inline pmd_t *pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr) { - unsigned long *ret = (unsigned long *)read_pda(pmd_quick); + unsigned long *ret; - if (__builtin_expect(ret != NULL, 1)) { - write_pda(pmd_quick, (unsigned long *)(*ret)); + if ((ret = (unsigned long *)pmd_quicklist) != NULL) { /* avoid likely/unlikely here */ + pmd_quicklist = (unsigned long *)(*ret); ret[0] = 0; - dec_pgcache_size(); + pgtable_cache_size--; } return (pmd_t *)ret; } @@ -73,88 +62,91 @@ return pmd; } - -static inline pgd_t *pgd_alloc_one_fast (void) +static inline pgd_t *get_pgd_slow(void) { - unsigned long *ret = read_pda(pgd_quick); - - if (__builtin_expect(ret != NULL, 1)) { - write_pda(pgd_quick,(unsigned long *)(*ret)); - ret[0] = 0; - dec_pgcache_size(); - } else - ret = NULL; - return (pgd_t *) ret; + pgd_t * pgd = (pgd_t *)__get_free_page(GFP_KERNEL); + if (__builtin_expect(pgd != NULL, 1)) + clear_page(pgd); + return pgd; } -static inline pgd_t *pgd_alloc (struct mm_struct *mm) +static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */ - pgd_t *pgd = pgd_alloc_one_fast(); + unsigned long *ret; - if (__builtin_expect(pgd == NULL, 0)) { - pgd = (pgd_t *)__get_free_page(GFP_KERNEL); - if (__builtin_expect(pgd != NULL, 1)) - clear_page(pgd); - } - return pgd; + if ((ret = pgd_quicklist) != NULL) { + pgd_quicklist = (unsigned long *)(*ret); + ret[0] = 0; + pgtable_cache_size--; + } else + ret = (unsigned long *)get_pgd_slow(); + return (pgd_t *)ret; } static inline void pgd_free (pgd_t *pgd) { - *(unsigned long *)pgd = (unsigned long) read_pda(pgd_quick); - write_pda(pgd_quick,(unsigned long *) pgd); - inc_pgcache_size(); + *(unsigned long *)pgd = (unsigned long) pgd_quicklist; + pgd_quicklist = (unsigned long *) pgd; + pgtable_cache_size++; } -static inline void pgd_free_slow (pgd_t *pgd) +static inline void free_pgd_slow (pgd_t *pgd) { - if ((unsigned long)pgd & (PAGE_SIZE-1)) - BUG(); +#ifdef CONFIG_CHECKING + if ((unsigned long)pgd & (PAGE_SIZE-1)) + BUG(); +#endif free_page((unsigned long)pgd); } -static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address) +static inline struct page * pte_alloc_one_fast(struct mm_struct *mm, + unsigned long address) { - pte_t *pte; + struct list_head * entry = pte_quicklist.next; /* FIFO */ + struct page * page = NULL; - pte = (pte_t *) __get_free_page(GFP_KERNEL); - if (pte) - clear_page(pte); - return pte; + if (entry != &pte_quicklist) { /* don't add a likely/unlikely here */ + list_del(entry); + page = list_entry(entry, struct page, list); + pgtable_cache_size--; + } + return page; } -extern __inline__ pte_t *pte_alloc_one_fast(struct mm_struct *mm, unsigned long address) +static inline struct page * pte_alloc_one_fast_lifo(struct mm_struct *mm, + unsigned long address) { - unsigned long *ret; + struct list_head * entry = pte_quicklist.prev; /* LIFO */ + struct page * page = NULL; - if(__builtin_expect((ret = read_pda(pte_quick)) != NULL, !0)) { - write_pda(pte_quick, (unsigned long *)(*ret)); - ret[0] = ret[1]; - dec_pgcache_size(); + if (entry != &pte_quicklist) { + list_del(entry); + page = list_entry(entry, struct page, list); + pgtable_cache_size--; } - return (pte_t *)ret; + return page; } /* Should really implement gc for free page table pages. This could be done with a reference count in struct page. */ -extern __inline__ void pte_free(pte_t *pte) +static inline void pte_free(struct page * page) { - *(unsigned long *)pte = (unsigned long) read_pda(pte_quick); - write_pda(pte_quick, (unsigned long *) pte); - inc_pgcache_size(); + list_add(&page->list, &pte_quicklist); + pgtable_cache_size++; } -extern __inline__ void pte_free_slow(pte_t *pte) +static __inline__ void pte_free_slow(struct page * page) { - if ((unsigned long)pte & (PAGE_SIZE-1)) - BUG(); - free_page((unsigned long)pte); + __free_page(page); } +static inline void pte_free_via_pmd(pmd_t pmd) +{ + pte_free(virt_to_page(pte_offset(&pmd, 0))); +} extern int do_check_pgt_cache(int, int); diff -urN x86-64-ref/include/asm-x86_64/pgtable.h x86-64/include/asm-x86_64/pgtable.h --- x86-64-ref/include/asm-x86_64/pgtable.h Tue Mar 12 10:42:25 2002 +++ x86-64/include/asm-x86_64/pgtable.h Tue Mar 12 09:45:04 2002 @@ -328,6 +328,14 @@ (unsigned long long)PAGE_SIZE + pgprot_val(pgprot))); \ __pte; \ }) +#define mk_pmd(page,pgprot) \ +({ \ + pmd_t __pmd; \ + \ + set_pmd(&__pmd, __pmd(((page)-mem_map) * \ + (unsigned long long)PAGE_SIZE + pgprot_val(pgprot))); \ + __pmd; \ +}) /* This takes a physical page address that is used by the remapping functions */ #define mk_pte_phys(physpage, pgprot) \ @@ -359,6 +367,13 @@ ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) #define pte_offset(dir, address) ((pte_t *) pmd_page(*(dir)) + \ __pte_offset(address)) + +#define pte_offset2(dir, address) pte_offset(dir, address) +#define pte_offset_atomic(dir, address) pte_offset(dir, address) +#define pte_offset_under_lock(dir, address, mm) pte_offset(dir, address) +#define pte_offset2_under_lock(dir, address, mm) pte_offset(dir, address) +#define pte_kunmap(ptep) do { } while(0) +#define pte_kunmap2(ptep) do { } while(0) /* never use these in the common code */ #define level4_page(level4) ((unsigned long) __va(level4_val(level4) & PAGE_MASK)) diff -urN x86-64-ref/include/asm-x86_64/processor.h x86-64/include/asm-x86_64/processor.h --- x86-64-ref/include/asm-x86_64/processor.h Tue Mar 12 10:42:25 2002 +++ x86-64/include/asm-x86_64/processor.h Tue Mar 12 09:45:04 2002 @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,11 @@ int f00f_bug; int coma_bug; unsigned long loops_per_jiffy; + + unsigned long *pgd_quick; + unsigned long *pmd_quick; + struct list_head pte_quick; + unsigned long pgtable_cache_sz; } ____cacheline_aligned; #define X86_VENDOR_INTEL 0