From 66508b9a0d3b979aaf24ad19ab20491fbb48eb63 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sun, 23 Jan 2011 19:17:40 -0500 Subject: [PATCH] page_alloc: augment percpu pages support for newer kernels This is a follow-on to: mm-page_alloc-rt-friendly-per-cpu-pages.patch (tip ff3fd6afd788760c846a2f4449487debb6c4b0ac) and: mm-page_alloc-reduce-lock-sections-further.patch (tip 46167aec68f48cbbeff23cae9173bc4d19a7bcda) which is embedded in the big merge up to 33rt (5f854cfc024) At some point it probably makes sense to munge all three of these together -- whee. See the constituent bits spread around in a git tip repo with: git diff 5f854cfc024622e4aae14d7cf422f6ff86278688^2 \ 5f854cfc024622e4aae14d7cf422f6ff86278688 mm/page_alloc.c (the above merge changeset is a superset of this delta). You can find the origin of this change in the tip merge commit: commit 5f854cfc024622e4aae14d7cf422f6ff86278688 Merge: cc24da0 4ec62b2 Author: Thomas Gleixner Date: Sun Feb 21 20:17:22 2010 +0100 Forward to 2.6.33-rc8 Merge branch 'linus' into rt/head with a pile of conflicts. Signed-off-by: Thomas Gleixner Normally there are not significant changes/additions in a merge commit that are not from any other "normal" commit. But in this case there are, so break them out into separate explicit commits. Signed-off-by: Paul Gortmaker --- mm/page_alloc.c | 90 +++++++++++++++++++++++++++++++------------------------ 1 files changed, 51 insertions(+), 39 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index fac0711..a81bf78 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -196,7 +196,7 @@ static inline void __lock_cpu_pcp(unsigned long *flags, int cpu) { #ifdef CONFIG_PREEMPT_RT spin_lock(&__get_cpu_lock(pcp_locks, cpu)); - flags = 0; + *flags = 0; #else local_irq_save(*flags); #endif @@ -595,10 +595,9 @@ static inline int free_pages_check(struct page *page) * pinned" detection logic. */ static void free_pcppages_bulk(struct zone *zone, int count, - struct per_cpu_pages *pcp) + struct per_cpu_pages *pcp) { int migratetype = 0; - int batch_free = 0; unsigned long flags; spin_lock_irqsave(&zone->lock, flags); @@ -606,6 +605,38 @@ static void free_pcppages_bulk(struct zone *zone, int count, zone->pages_scanned = 0; __mod_zone_page_state(zone, NR_FREE_PAGES, count); + + for (migratetype =0; migratetype < MIGRATE_PCPTYPES; migratetype++) { + struct list_head *list = &pcp->lists[migratetype]; + + while (!list_empty(list)) { + struct page *page; + + page = list_first_entry(list, struct page, lru); + /* must delete as __free_one_page list manipulates */ + list_del(&page->lru); + /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ + __free_one_page(page, zone, 0, page_private(page)); + trace_mm_page_pcpu_drain(page, 0, page_private(page)); +#ifdef CONFIG_PREEMPT_RT + cond_resched_lock(&zone->lock); +#endif + count--; + } + } + WARN_ON(count != 0); + spin_unlock_irqrestore(&zone->lock, flags); +} + +static void isolate_pcp_pages(int count, struct per_cpu_pages *src, + struct per_cpu_pages *dst) +{ + int migratetype, batch_free = 0; + + for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) + INIT_LIST_HEAD(&dst->lists[migratetype]); + migratetype = 0; + while (count) { struct page *page; struct list_head *list; @@ -621,22 +652,16 @@ static void free_pcppages_bulk(struct zone *zone, int count, batch_free++; if (++migratetype == MIGRATE_PCPTYPES) migratetype = 0; - list = &pcp->lists[migratetype]; + list = &src->lists[migratetype]; } while (list_empty(list)); do { - page = list_entry(list->prev, struct page, lru); + page = list_last_entry(list, struct page, lru); /* must delete as __free_one_page list manipulates */ list_del(&page->lru); - /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ - __free_one_page(page, zone, 0, page_private(page)); - trace_mm_page_pcpu_drain(page, 0, page_private(page)); -#ifdef CONFIG_PREEMPT_RT - cond_resched_lock(&zone->lock); -#endif + list_add(&page->lru, &dst->lists[migratetype]); } while (--count && --batch_free && !list_empty(list)); } - spin_unlock_irqrestore(&zone->lock, flags); } static void free_one_page(struct zone *zone, struct page *page, int order, @@ -1045,16 +1070,6 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, return i; } -static void -isolate_pcp_pages(int count, struct list_head *src, struct list_head *dst) -{ - while (count--) { - struct page *page = list_last_entry(src, struct page, lru); - list_move(&page->lru, dst); - } -} - - #ifdef CONFIG_NUMA /* * Called from the vmstat counter updater to drain pagesets of this @@ -1066,20 +1081,19 @@ isolate_pcp_pages(int count, struct list_head *src, struct list_head *dst) */ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) { - LIST_HEAD(free_list); + struct per_cpu_pages dst; unsigned long flags; - int to_drain; - int this_cpu; + int to_drain, this_cpu; lock_cpu_pcp(&flags, &this_cpu); if (pcp->count >= pcp->batch) to_drain = pcp->batch; else to_drain = pcp->count; - isolate_pcp_pages(to_drain, &pcp->list, &free_list); + isolate_pcp_pages(to_drain, pcp, &dst); pcp->count -= to_drain; unlock_cpu_pcp(flags, this_cpu); - free_pages_bulk(zone, to_drain, &free_list, 0); + free_pcppages_bulk(zone, to_drain, &dst); } #endif @@ -1097,8 +1111,7 @@ static void drain_pages(unsigned int cpu) for_each_populated_zone(zone) { struct per_cpu_pageset *pset; - struct per_cpu_pages *pcp; - LIST_HEAD(free_list); + struct per_cpu_pages *pcp, dst; int count; __lock_cpu_pcp(&flags, cpu); @@ -1109,12 +1122,13 @@ static void drain_pages(unsigned int cpu) WARN_ON(1); continue; } + pcp = &pset->pcp; - isolate_pcp_pages(pcp->count, &pcp->list, &free_list); + isolate_pcp_pages(pcp->count, pcp, &dst); count = pcp->count; pcp->count = 0; unlock_cpu_pcp(flags, cpu); - free_pages_bulk(zone, count, &free_list, 0); + free_pcppages_bulk(zone, count, &dst); } } @@ -1220,8 +1234,8 @@ void free_hot_cold_page(struct page *page, int cold) struct per_cpu_pageset *pset; struct per_cpu_pages *pcp; unsigned long flags; - int migratetype; - int count, this_cpu, wasMlocked = __TestClearPageMlocked(page); + int migratetype, this_cpu, count; + int wasMlocked = __TestClearPageMlocked(page); trace_mm_page_free_direct(page, 0); kmemcheck_free_shadow(page, 0); @@ -1267,16 +1281,15 @@ void free_hot_cold_page(struct page *page, int cold) list_add(&page->lru, &pcp->lists[migratetype]); pcp->count++; if (pcp->count >= pcp->high) { - LIST_HEAD(free_list); + struct per_cpu_pages dst; - isolate_pcp_pages(pcp->batch, &pcp->list, &free_list); + isolate_pcp_pages(pcp->batch, pcp, &dst); pcp->count -= pcp->batch; count = pcp->batch; put_zone_pcp(zone, flags, this_cpu); - free_pages_bulk(zone, count, &free_list, 0); + free_pcppages_bulk(zone, count, &dst); return; } - out: put_zone_pcp(zone, flags, this_cpu); } @@ -1329,11 +1342,10 @@ again: pset = get_zone_pcp(zone, &flags, &this_cpu); if (likely(order == 0)) { - struct list_head *list; struct per_cpu_pages *pcp = &pset->pcp; + struct list_head *list; list = &pcp->lists[migratetype]; - if (list_empty(list)) { pcp->count += rmqueue_bulk(zone, 0, pcp->batch, list, -- 1.7.0.4