[PATCH] UML: Three-level page table support

This is the three-level page table support from the x86_64 patch.  It can
be enabled on x86, although it's not particularly needed at this point.
However, it can be used to implement very large physical memory (with
almost all of it in highmem) on UML.

Signed-off-by: Jeff Dike <jdike@addtoit.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

9 files changed, 246 insertions, 89 deletions

diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index 06ba2aca73a54c..633e8a54fde297 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -68,6 +68,8 @@ config MODE_SKAS
 	to CONFIG_MODE_TT).  Otherwise, it is safe to say Y.  Disabling this
 	option will shrink the UML binary slightly.
 
+source "arch/um/Kconfig_arch"
+
 config NET
 	bool "Networking support"
 	help
diff --git a/arch/um/Kconfig_i386 b/arch/um/Kconfig_i386
new file mode 100644
index 00000000000000..e2a5f6692cd678
--- /dev/null
+++ b/arch/um/Kconfig_i386
@@ -0,0 +1,8 @@
+config 3_LEVEL_PGTABLES
+	bool "Three-level pagetables"
+	default n
+	help
+	Three-level pagetables will let UML have more than 4G of physical
+	memory.  All the memory that can't be mapped directly will be treated
+	as high memory.
+
diff --git a/arch/um/Makefile b/arch/um/Makefile
index b8371a95849a05..288a375b9c3b24 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -17,7 +17,7 @@ core-y			+= $(ARCH_DIR)/kernel/		\
 
 # Have to precede the include because the included Makefiles reference them.
 SYMLINK_HEADERS = archparam.h system.h sigcontext.h processor.h ptrace.h \
-	arch-signal.h module.h
+	arch-signal.h module.h vm-flags.h
 SYMLINK_HEADERS := $(foreach header,$(SYMLINK_HEADERS),include/asm-um/$(header))
 
 ARCH_SYMLINKS = include/asm-um/arch $(ARCH_DIR)/include/sysdep $(ARCH_DIR)/os \
diff --git a/arch/um/defconfig b/arch/um/defconfig
index 708878e397a918..51558110902949 100644
--- a/arch/um/defconfig
+++ b/arch/um/defconfig
@@ -17,6 +17,7 @@ CONFIG_GENERIC_CALIBRATE_DELAY=y
 #
 CONFIG_MODE_TT=y
 CONFIG_MODE_SKAS=y
+# CONFIG_3_LEVEL_PGTABLES is not set
 CONFIG_NET=y
 CONFIG_BINFMT_ELF=y
 CONFIG_BINFMT_MISC=m
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 77d063d5407f99..e8a2c9c5716b40 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -25,7 +25,7 @@ extern char __binary_start;
 /* Changed during early boot */
 unsigned long *empty_zero_page = NULL;
 unsigned long *empty_bad_page = NULL;
-pgd_t swapper_pg_dir[1024];
+pgd_t swapper_pg_dir[PTRS_PER_PGD];
 unsigned long highmem;
 int kmalloc_ok = 0;
 
@@ -242,6 +242,7 @@ struct page *arch_validate(struct page *page, int mask, int order)
 		}
 		addr += PAGE_SIZE;
 	}
+
 	if(i == (1 << order)) return(page);
 	page = alloc_pages(mask, order);
 	goto again;
@@ -336,7 +337,7 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
 {
 	struct page *pte;
    
-	pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
+	pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
 	return pte;
 }
 
diff --git a/arch/um/kernel/physmem.c b/arch/um/kernel/physmem.c
index 3253bc0e3a5cad..2853028657b36b 100644
--- a/arch/um/kernel/physmem.c
+++ b/arch/um/kernel/physmem.c
@@ -309,7 +309,7 @@ struct page *__virt_to_page(const unsigned long virt)
 	return(&mem_map[__pa(virt) >> PAGE_SHIFT]);
 }
 
-unsigned long page_to_phys(struct page *page)
+phys_t page_to_phys(struct page *page)
 {
 	return((page - mem_map) << PAGE_SHIFT);
 }
@@ -318,8 +318,9 @@ pte_t mk_pte(struct page *page, pgprot_t pgprot)
 {
 	pte_t pte;
 
-	pte_val(pte) = page_to_phys(page) + pgprot_val(pgprot);
-	if(pte_present(pte)) pte_mknewprot(pte_mknewpage(pte));
+	pte_set_val(pte, page_to_phys(page), pgprot);
+	if(pte_present(pte))
+		pte_mknewprot(pte_mknewpage(pte));
 	return(pte);
 }
 
diff --git a/arch/um/kernel/skas/tlb.c b/arch/um/kernel/skas/tlb.c
index 3b0d8da430c194..956fb0102a1ee4 100644
--- a/arch/um/kernel/skas/tlb.c
+++ b/arch/um/kernel/skas/tlb.c
@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2002 Jeff Dike (jdike@karaya.com)
+ * Copyright 2003 PathScale, Inc.
  * Licensed under the GPL
  */
 
@@ -18,54 +19,86 @@ static void fix_range(struct mm_struct *mm, unsigned long start_addr,
 		      unsigned long end_addr, int force)
 {
 	pgd_t *npgd;
-	pmd_t *npud;
+	pud_t *npud;
 	pmd_t *npmd;
 	pte_t *npte;
-	unsigned long addr;
+	unsigned long addr,  end;
 	int r, w, x, err, fd;
 
 	if(mm == NULL) return;
 	fd = mm->context.skas.mm_fd;
 	for(addr = start_addr; addr < end_addr;){
 		npgd = pgd_offset(mm, addr);
-		npud = pud_offset(npgd, addr);
-		npmd = pmd_offset(npud, addr);
-		if(pmd_present(*npmd)){
-			npte = pte_offset_kernel(npmd, addr);
-			r = pte_read(*npte);
-			w = pte_write(*npte);
-			x = pte_exec(*npte);
-			if(!pte_dirty(*npte)) w = 0;
-			if(!pte_young(*npte)){
-				r = 0;
-				w = 0;
-			}
-			if(force || pte_newpage(*npte)){
-				err = unmap(fd, (void *) addr, PAGE_SIZE);
+		if(!pgd_present(*npgd)){
+			if(force || pgd_newpage(*npgd)){
+				end = addr + PGDIR_SIZE;
+				if(end > end_addr)
+					end = end_addr;
+				err = unmap(fd, (void *) addr, end - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
-				if(pte_present(*npte))
-					map(fd, addr, 
-					    pte_val(*npte) & PAGE_MASK,
-					    PAGE_SIZE, r, w, x);
+				pgd_mkuptodate(*npgd);
 			}
-			else if(pte_newprot(*npte)){
-				protect(fd, addr, PAGE_SIZE, r, w, x, 1);
+			addr += PGDIR_SIZE;
+			continue;
+		}
+
+		npud = pud_offset(npgd, addr);
+		if(!pud_present(*npud)){
+			if(force || pud_newpage(*npud)){
+				end = addr + PUD_SIZE;
+				if(end > end_addr)
+					end = end_addr;
+				err = unmap(fd, (void *) addr, end - addr);
+				if(err < 0)
+					panic("munmap failed, errno = %d\n",
+					      -err);
+				pud_mkuptodate(*npud);
 			}
-			*npte = pte_mkuptodate(*npte);
-			addr += PAGE_SIZE;
+			addr += PUD_SIZE;
+			continue;
 		}
-		else {
+
+		npmd = pmd_offset(npud, addr);
+		if(!pmd_present(*npmd)){
 			if(force || pmd_newpage(*npmd)){
-				err = unmap(fd, (void *) addr, PMD_SIZE);
+				end = addr + PMD_SIZE;
+				if(end > end_addr)
+					end = end_addr;
+				err = unmap(fd, (void *) addr, end - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
 				pmd_mkuptodate(*npmd);
 			}
 			addr += PMD_SIZE;
+			continue;
+		}
+
+		npte = pte_offset_kernel(npmd, addr);
+		r = pte_read(*npte);
+		w = pte_write(*npte);
+		x = pte_exec(*npte);
+		if(!pte_dirty(*npte))
+			w = 0;
+		if(!pte_young(*npte)){
+			r = 0;
+			w = 0;
 		}
+		if(force || pte_newpage(*npte)){
+			err = unmap(fd, (void *) addr, PAGE_SIZE);
+			if(err < 0)
+				panic("munmap failed, errno = %d\n", -err);
+			if(pte_present(*npte))
+				map(fd, addr, pte_val(*npte) & PAGE_MASK,
+				    PAGE_SIZE, r, w, x);
+		}
+		else if(pte_newprot(*npte))
+			protect(fd, addr, PAGE_SIZE, r, w, x, 1);
+
+		*npte = pte_mkuptodate(*npte);
+		addr += PAGE_SIZE;
 	}
 }
 
@@ -73,9 +106,10 @@ void flush_tlb_kernel_range_skas(unsigned long start, unsigned long end)
 {
 	struct mm_struct *mm;
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
-	unsigned long addr;
+	unsigned long addr, last;
 	int updated = 0, err;
 
 	mm = &init_mm;
@@ -83,36 +117,71 @@ void flush_tlb_kernel_range_skas(unsigned long start, unsigned long end)
 		pgd = pgd_offset(mm, addr);
 		pud = pud_offset(pgd, addr);
 		pmd = pmd_offset(pud, addr);
-		if(pmd_present(*pmd)){
-			pte = pte_offset_kernel(pmd, addr);
-			if(!pte_present(*pte) || pte_newpage(*pte)){
+ 		if(!pgd_present(*pgd)){
+			if(pgd_newpage(*pgd)){
 				updated = 1;
+				last = addr + PGDIR_SIZE;
+				if(last > end)
+					last = end;
 				err = os_unmap_memory((void *) addr, 
-						      PAGE_SIZE);
+						      last - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
-				if(pte_present(*pte))
-					map_memory(addr, 
-						   pte_val(*pte) & PAGE_MASK,
-						   PAGE_SIZE, 1, 1, 1);
 			}
-			else if(pte_newprot(*pte)){
+			addr += PGDIR_SIZE;
+			continue;
+		}
+
+		pud = pud_offset(pgd, addr);
+		if(!pud_present(*pud)){
+			if(pud_newpage(*pud)){
 				updated = 1;
-				protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1);
+				last = addr + PUD_SIZE;
+				if(last > end)
+					last = end;
+				err = os_unmap_memory((void *) addr,
+						      last - addr);
+				if(err < 0)
+					panic("munmap failed, errno = %d\n",
+					      -err);
 			}
-			addr += PAGE_SIZE;
+			addr += PUD_SIZE;
+			continue;
 		}
-		else {
+
+		pmd = pmd_offset(pud, addr);
+		if(!pmd_present(*pmd)){
 			if(pmd_newpage(*pmd)){
 				updated = 1;
-				err = os_unmap_memory((void *) addr, PMD_SIZE);
+				last = addr + PMD_SIZE;
+				if(last > end)
+					last = end;
+				err = os_unmap_memory((void *) addr,
+						      last - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
 			}
 			addr += PMD_SIZE;
+			continue;
+		}
+
+		pte = pte_offset_kernel(pmd, addr);
+		if(!pte_present(*pte) || pte_newpage(*pte)){
+			updated = 1;
+			err = os_unmap_memory((void *) addr, PAGE_SIZE);
+			if(err < 0)
+				panic("munmap failed, errno = %d\n", -err);
+			if(pte_present(*pte))
+				map_memory(addr, pte_val(*pte) & PAGE_MASK,
+					   PAGE_SIZE, 1, 1, 1);
 		}
+		else if(pte_newprot(*pte)){
+			updated = 1;
+			protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1);
+  		}
+		addr += PAGE_SIZE;
 	}
 }
 
diff --git a/arch/um/kernel/trap_kern.c b/arch/um/kernel/trap_kern.c
index 9385a1aefaba6f..80d07fe6b97923 100644
--- a/arch/um/kernel/trap_kern.c
+++ b/arch/um/kernel/trap_kern.c
@@ -56,7 +56,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
 	if(is_write && !(vma->vm_flags & VM_WRITE)) 
 		goto out;
 	page = address & PAGE_MASK;
-	pgd = pgd_offset(mm);
+	pgd = pgd_offset(mm, page);
 	pud = pud_offset(pgd, page);
 	pmd = pmd_offset(pud, page);
 	do {
@@ -77,6 +77,9 @@ int handle_page_fault(unsigned long address, unsigned long ip,
 		default:
 			BUG();
 		}
+		pgd = pgd_offset(mm, page);
+		pud = pud_offset(pgd, page);
+		pmd = pmd_offset(pud, page);
 		pte = pte_offset_kernel(pmd, page);
 	} while(!pte_present(*pte));
 	err = 0;
diff --git a/arch/um/kernel/tt/tlb.c b/arch/um/kernel/tt/tlb.c
index ce057e3c1fb41c..3bc35eea1da4ad 100644
--- a/arch/um/kernel/tt/tlb.c
+++ b/arch/um/kernel/tt/tlb.c
@@ -1,5 +1,6 @@
 /* 
  * Copyright (C) 2002 Jeff Dike (jdike@karaya.com)
+ * Copyright 2003 PathScale, Inc.
  * Licensed under the GPL
  */
 
@@ -22,7 +23,7 @@ static void fix_range(struct mm_struct *mm, unsigned long start_addr,
 	pud_t *npud;
 	pmd_t *npmd;
 	pte_t *npte;
-	unsigned long addr;
+	unsigned long addr, end;
 	int r, w, x, err;
 
 	if((current->thread.mode.tt.extern_pid != -1) && 
@@ -42,46 +43,81 @@ static void fix_range(struct mm_struct *mm, unsigned long start_addr,
 			addr = STACK_TOP - ABOVE_KMEM;
 			continue;
 		}
+
 		npgd = pgd_offset(mm, addr);
+ 		if(!pgd_present(*npgd)){
+ 			if(force || pgd_newpage(*npgd)){
+ 				end = addr + PGDIR_SIZE;
+ 				if(end > end_addr)
+ 					end = end_addr;
+				err = os_unmap_memory((void *) addr,
+ 						      end - addr);
+				if(err < 0)
+					panic("munmap failed, errno = %d\n",
+					      -err);
+				pgd_mkuptodate(*npgd);
+ 			}
+			addr += PGDIR_SIZE;
+			continue;
+ 		}
+
 		npud = pud_offset(npgd, addr);
-		npmd = pmd_offset(npud, addr);
-		if(pmd_present(*npmd)){
-			npte = pte_offset_kernel(npmd, addr);
-			r = pte_read(*npte);
-			w = pte_write(*npte);
-			x = pte_exec(*npte);
-			if(!pte_dirty(*npte)) w = 0;
-			if(!pte_young(*npte)){
-				r = 0;
-				w = 0;
-			}
-			if(force || pte_newpage(*npte)){
+		if(!pud_present(*npud)){
+			if(force || pud_newpage(*npud)){
+ 				end = addr + PUD_SIZE;
+ 				if(end > end_addr)
+ 					end = end_addr;
 				err = os_unmap_memory((void *) addr, 
-						      PAGE_SIZE);
+						      end - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
-				if(pte_present(*npte))
-					map_memory(addr, 
-						   pte_val(*npte) & PAGE_MASK,
-						   PAGE_SIZE, r, w, x);
-			}
-			else if(pte_newprot(*npte)){
-				protect_memory(addr, PAGE_SIZE, r, w, x, 1);
+				pud_mkuptodate(*npud);
 			}
-			*npte = pte_mkuptodate(*npte);
-			addr += PAGE_SIZE;
+			addr += PUD_SIZE;
+			continue;
 		}
-		else {
+
+		npmd = pmd_offset(npud, addr);
+		if(!pmd_present(*npmd)){
 			if(force || pmd_newpage(*npmd)){
-				err = os_unmap_memory((void *) addr, PMD_SIZE);
+ 				end = addr + PMD_SIZE;
+ 				if(end > end_addr)
+ 					end = end_addr;
+				err = os_unmap_memory((void *) addr,
+						      end - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
 				pmd_mkuptodate(*npmd);
 			}
 			addr += PMD_SIZE;
+			continue;
 		}
+
+		npte = pte_offset_kernel(npmd, addr);
+		r = pte_read(*npte);
+		w = pte_write(*npte);
+		x = pte_exec(*npte);
+		if(!pte_dirty(*npte))
+			w = 0;
+		if(!pte_young(*npte)){
+			r = 0;
+			w = 0;
+		}
+		if(force || pte_newpage(*npte)){
+			err = os_unmap_memory((void *) addr, PAGE_SIZE);
+			if(err < 0)
+				panic("munmap failed, errno = %d\n", -err);
+			if(pte_present(*npte))
+				map_memory(addr, pte_val(*npte) & PAGE_MASK,
+					   PAGE_SIZE, r, w, x);
+		}
+		else if(pte_newprot(*npte))
+			protect_memory(addr, PAGE_SIZE, r, w, x, 1);
+
+		*npte = pte_mkuptodate(*npte);
+		addr += PAGE_SIZE;
 	}
 }
 
@@ -92,47 +128,83 @@ static void flush_kernel_vm_range(unsigned long start, unsigned long end,
 {
 	struct mm_struct *mm;
 	pgd_t *pgd;
-	pud_t *pmd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
-	unsigned long addr;
+	unsigned long addr, last;
 	int updated = 0, err;
 
 	mm = &init_mm;
 	for(addr = start; addr < end;){
 		pgd = pgd_offset(mm, addr);
-		pud = pud_offset(pgd, addr);
-		pmd = pmd_offset(pud, addr);
-		if(pmd_present(*pmd)){
-			pte = pte_offset_kernel(pmd, addr);
-			if(!pte_present(*pte) || pte_newpage(*pte)){
+ 		if(!pgd_present(*pgd)){
+ 			if(pgd_newpage(*pgd)){
 				updated = 1;
+ 				last = addr + PGDIR_SIZE;
+ 				if(last > end)
+ 					last = end;
 				err = os_unmap_memory((void *) addr, 
-						      PAGE_SIZE);
+						      last - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
-				if(pte_present(*pte))
-					map_memory(addr, 
-						   pte_val(*pte) & PAGE_MASK,
-						   PAGE_SIZE, 1, 1, 1);
 			}
-			else if(pte_newprot(*pte)){
+			addr += PGDIR_SIZE;
+			continue;
+		}
+
+		pud = pud_offset(pgd, addr);
+		if(!pud_present(*pud)){
+			if(pud_newpage(*pud)){
 				updated = 1;
-				protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1);
+				last = addr + PUD_SIZE;
+				if(last > end)
+					last = end;
+				err = os_unmap_memory((void *) addr,
+						      last - addr);
+				if(err < 0)
+					panic("munmap failed, errno = %d\n",
+					      -err);
 			}
-			addr += PAGE_SIZE;
+			addr += PUD_SIZE;
+			continue;
 		}
-		else {
+
+		pmd = pmd_offset(pud, addr);
+		if(!pmd_present(*pmd)){
 			if(pmd_newpage(*pmd)){
 				updated = 1;
-				err = os_unmap_memory((void *) addr, PMD_SIZE);
+				last = addr + PMD_SIZE;
+				if(last > end)
+					last = end;
+				err = os_unmap_memory((void *) addr,
+						      last - addr);
 				if(err < 0)
 					panic("munmap failed, errno = %d\n",
 					      -err);
 			}
 			addr += PMD_SIZE;
+			continue;
+		}
+
+		pte = pte_offset_kernel(pmd, addr);
+		if(!pte_present(*pte) || pte_newpage(*pte)){
+			updated = 1;
+			err = os_unmap_memory((void *) addr,
+					      PAGE_SIZE);
+			if(err < 0)
+				panic("munmap failed, errno = %d\n",
+				      -err);
+			if(pte_present(*pte))
+				map_memory(addr,
+					   pte_val(*pte) & PAGE_MASK,
+					   PAGE_SIZE, 1, 1, 1);
+		}
+		else if(pte_newprot(*pte)){
+			updated = 1;
+			protect_memory(addr, PAGE_SIZE, 1, 1, 1, 1);
 		}
+		addr += PAGE_SIZE;
 	}
 	if(updated && update_seq) atomic_inc(&vmchange_seq);
 }