[PATCH] rmaplock: PageAnon in mapping

First of a batch of five patches to eliminate rmap's page_map_lock, replace
its trylocking by spinlocking, and use anon_vma to speed up swapoff.

Patches updated from the originals against 2.6.7-mm7: nothing new so I won't
spam the list, but including Manfred's SLAB_DESTROY_BY_RCU fixes, and omitting
the unuse_process mmap_sem fix already in 2.6.8-rc3.


This patch:

Replace the PG_anon page->flags bit by setting the lower bit of the pointer in
page->mapping when it's anon_vma: PAGE_MAPPING_ANON bit.

We're about to eliminate the locking which kept the flags and mapping in
synch: it's much easier to work on a local copy of page->mapping, than worry
about whether flags and mapping are in synch (though I imagine it could be
done, at greater cost, with some barriers).

Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

2 files changed, 3 insertions, 20 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index aa5cfa69e584a5..01812f28bd23cc 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -88,7 +88,6 @@ static void bad_page(const char *function, struct page *page)
 			1 << PG_active	|
 			1 << PG_dirty	|
 			1 << PG_maplock |
-			1 << PG_anon    |
 			1 << PG_swapcache |
 			1 << PG_writeback);
 	set_page_count(page, 0);
@@ -230,7 +229,6 @@ static inline void free_pages_check(const char *function, struct page *page)
 			1 << PG_reclaim	|
 			1 << PG_slab	|
 			1 << PG_maplock |
-			1 << PG_anon    |
 			1 << PG_swapcache |
 			1 << PG_writeback )))
 		bad_page(function, page);
@@ -353,7 +351,6 @@ static void prep_new_page(struct page *page, int order)
 			1 << PG_dirty	|
 			1 << PG_reclaim	|
 			1 << PG_maplock |
-			1 << PG_anon    |
 			1 << PG_swapcache |
 			1 << PG_writeback )))
 		bad_page(__FUNCTION__, page);
diff --git a/mm/rmap.c b/mm/rmap.c
index 1cb3353daa16d3..a7c1ae8ff4048e 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -168,7 +168,6 @@ static inline void clear_page_anon(struct page *page)
 {
 	BUG_ON(!page->mapping);
 	page->mapping = NULL;
-	ClearPageAnon(page);
 }
 
 /*
@@ -246,7 +245,7 @@ out:
 static inline int page_referenced_anon(struct page *page)
 {
 	unsigned int mapcount = page->mapcount;
-	struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+	struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
 	struct vm_area_struct *vma;
 	int referenced = 0;
 
@@ -346,31 +345,18 @@ void page_add_anon_rmap(struct page *page,
 	BUG_ON(PageReserved(page));
 	BUG_ON(!anon_vma);
 
+	anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
 	index = (address - vma->vm_start) >> PAGE_SHIFT;
 	index += vma->vm_pgoff;
 	index >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
 
-	/*
-	 * Setting and clearing PG_anon must always happen inside
-	 * page_map_lock to avoid races between mapping and
-	 * unmapping on different processes of the same
-	 * shared cow swapcache page. And while we take the
-	 * page_map_lock PG_anon cannot change from under us.
-	 * Actually PG_anon cannot change under fork either
-	 * since fork holds a reference on the page so it cannot
-	 * be unmapped under fork and in turn copy_page_range is
-	 * allowed to read PG_anon outside the page_map_lock.
-	 */
 	page_map_lock(page);
 	if (!page->mapcount) {
-		BUG_ON(PageAnon(page));
 		BUG_ON(page->mapping);
-		SetPageAnon(page);
 		page->index = index;
 		page->mapping = (struct address_space *) anon_vma;
 		inc_page_state(nr_mapped);
 	} else {
-		BUG_ON(!PageAnon(page));
 		BUG_ON(page->index != index);
 		BUG_ON(page->mapping != (struct address_space *) anon_vma);
 	}
@@ -629,7 +615,7 @@ out_unlock:
 
 static inline int try_to_unmap_anon(struct page *page)
 {
-	struct anon_vma *anon_vma = (struct anon_vma *) page->mapping;
+	struct anon_vma *anon_vma = (void *) page->mapping - PAGE_MAPPING_ANON;
 	struct vm_area_struct *vma;
 	int ret = SWAP_AGAIN;