From: Pavel Machek Some machines are spending minutes of CPU time during suspend in stupid O(n^2) algorithm. This patch replaces it with O(n) algorithm, making swsusp usable to some people. Signed-off-by: Pavel Machek Signed-off-by: Andrew Morton --- 25-akpm/include/linux/page-flags.h | 5 + 25-akpm/include/linux/suspend.h | 1 25-akpm/kernel/power/swsusp.c | 131 +++++++++++++++---------------------- 25-akpm/mm/page_alloc.c | 30 ++++---- 4 files changed, 79 insertions(+), 88 deletions(-) diff -puN include/linux/page-flags.h~swsusp-kill-on2-algorithm-in-swsusp include/linux/page-flags.h --- 25/include/linux/page-flags.h~swsusp-kill-on2-algorithm-in-swsusp 2005-01-01 21:25:56.112877664 -0800 +++ 25-akpm/include/linux/page-flags.h 2005-01-01 21:26:39.673255480 -0800 @@ -74,6 +74,7 @@ #define PG_swapcache 16 /* Swap page: swp_entry_t in private */ #define PG_mappedtodisk 17 /* Has blocks allocated on-disk */ #define PG_reclaim 18 /* To be reclaimed asap */ +#define PG_nosave_free 19 /* Free, should not be written */ /* @@ -277,6 +278,10 @@ extern unsigned long __read_page_state(u #define ClearPageNosave(page) clear_bit(PG_nosave, &(page)->flags) #define TestClearPageNosave(page) test_and_clear_bit(PG_nosave, &(page)->flags) +#define PageNosaveFree(page) test_bit(PG_nosave_free, &(page)->flags) +#define SetPageNosaveFree(page) set_bit(PG_nosave_free, &(page)->flags) +#define ClearPageNosaveFree(page) clear_bit(PG_nosave_free, &(page)->flags) + #define PageMappedToDisk(page) test_bit(PG_mappedtodisk, &(page)->flags) #define SetPageMappedToDisk(page) set_bit(PG_mappedtodisk, &(page)->flags) #define ClearPageMappedToDisk(page) clear_bit(PG_mappedtodisk, &(page)->flags) diff -puN include/linux/suspend.h~swsusp-kill-on2-algorithm-in-swsusp include/linux/suspend.h --- 25/include/linux/suspend.h~swsusp-kill-on2-algorithm-in-swsusp 2005-01-01 21:25:56.113877512 -0800 +++ 25-akpm/include/linux/suspend.h 2005-01-01 21:25:56.121876296 -0800 @@ -31,6 +31,7 @@ extern int shrink_mem(void); /* mm/page_alloc.c */ extern void drain_local_pages(void); +extern void mark_free_pages(struct zone *zone); /* kernel/power/swsusp.c */ extern int software_suspend(void); diff -puN kernel/power/swsusp.c~swsusp-kill-on2-algorithm-in-swsusp kernel/power/swsusp.c --- 25/kernel/power/swsusp.c~swsusp-kill-on2-algorithm-in-swsusp 2005-01-01 21:25:56.115877208 -0800 +++ 25-akpm/kernel/power/swsusp.c 2005-01-01 21:25:56.123875992 -0800 @@ -75,11 +75,9 @@ /* References to section boundaries */ extern const void __nosave_begin, __nosave_end; -extern int is_head_of_free_region(struct page *); - /* Variables to be preserved over suspend */ -int pagedir_order_check; -int nr_copy_pages_check; +static int pagedir_order_check; +static int nr_copy_pages_check; extern char resume_file[]; static dev_t resume_device; @@ -427,12 +425,12 @@ struct highmem_page *highmem_copy = NULL static int save_highmem_zone(struct zone *zone) { unsigned long zone_pfn; + mark_free_pages(zone); for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { struct page *page; struct highmem_page *save; void *kaddr; unsigned long pfn = zone_pfn + zone->zone_start_pfn; - int chunk_size; if (!(pfn%1000)) printk("."); @@ -449,11 +447,9 @@ static int save_highmem_zone(struct zone printk("highmem reserved page?!\n"); continue; } - if ((chunk_size = is_head_of_free_region(page))) { - pfn += chunk_size - 1; - zone_pfn += chunk_size - 1; + BUG_ON(PageNosave(page)); + if (PageNosaveFree(page)) continue; - } save = kmalloc(sizeof(struct highmem_page), GFP_ATOMIC); if (!save) return -ENOMEM; @@ -525,21 +521,16 @@ static int pfn_is_nosave(unsigned long p * We save a page if it's Reserved, and not in the range of pages * statically defined as 'unsaveable', or if it isn't reserved, and * isn't part of a free chunk of pages. - * If it is part of a free chunk, we update @pfn to point to the last - * page of the chunk. */ static int saveable(struct zone * zone, unsigned long * zone_pfn) { unsigned long pfn = *zone_pfn + zone->zone_start_pfn; - unsigned long chunk_size; struct page * page; if (!pfn_valid(pfn)) return 0; - if (!(pfn%1000)) - printk("."); page = pfn_to_page(pfn); BUG_ON(PageReserved(page) && PageNosave(page)); if (PageNosave(page)) @@ -548,10 +539,8 @@ static int saveable(struct zone * zone, pr_debug("[nosave pfn 0x%lx]", pfn); return 0; } - if ((chunk_size = is_head_of_free_region(page))) { - *zone_pfn += chunk_size - 1; + if (PageNosaveFree(page)) return 0; - } return 1; } @@ -564,10 +553,11 @@ static void count_data_pages(void) nr_copy_pages = 0; for_each_zone(zone) { - if (!is_highmem(zone)) { - for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) - nr_copy_pages += saveable(zone, &zone_pfn); - } + if (is_highmem(zone)) + continue; + mark_free_pages(zone); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) + nr_copy_pages += saveable(zone, &zone_pfn); } } @@ -577,52 +567,25 @@ static void copy_data_pages(void) struct zone *zone; unsigned long zone_pfn; struct pbe * pbe = pagedir_nosave; + int to_copy = nr_copy_pages; for_each_zone(zone) { - if (!is_highmem(zone)) - for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { - if (saveable(zone, &zone_pfn)) { - struct page * page; - page = pfn_to_page(zone_pfn + zone->zone_start_pfn); - pbe->orig_address = (long) page_address(page); - /* copy_page is no usable for copying task structs. */ - memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE); - pbe++; - } - } - } -} - - -static void free_suspend_pagedir_zone(struct zone *zone, unsigned long pagedir) -{ - unsigned long zone_pfn, pagedir_end, pagedir_pfn, pagedir_end_pfn; - pagedir_end = pagedir + (PAGE_SIZE << pagedir_order); - pagedir_pfn = __pa(pagedir) >> PAGE_SHIFT; - pagedir_end_pfn = __pa(pagedir_end) >> PAGE_SHIFT; - for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { - struct page *page; - unsigned long pfn = zone_pfn + zone->zone_start_pfn; - if (!pfn_valid(pfn)) - continue; - page = pfn_to_page(pfn); - if (!TestClearPageNosave(page)) - continue; - else if (pfn >= pagedir_pfn && pfn < pagedir_end_pfn) + if (is_highmem(zone)) continue; - __free_page(page); - } -} - -void swsusp_free(void) -{ - unsigned long p = (unsigned long)pagedir_save; - struct zone *zone; - for_each_zone(zone) { - if (!is_highmem(zone)) - free_suspend_pagedir_zone(zone, p); + mark_free_pages(zone); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) { + if (saveable(zone, &zone_pfn)) { + struct page * page; + page = pfn_to_page(zone_pfn + zone->zone_start_pfn); + pbe->orig_address = (long) page_address(page); + /* copy_page is not usable for copying task structs. */ + memcpy((void *)pbe->address, (void *)pbe->orig_address, PAGE_SIZE); + pbe++; + to_copy--; + } + } } - free_pages(p, pagedir_order); + BUG_ON(to_copy); } @@ -688,6 +651,24 @@ static int alloc_pagedir(void) return 0; } +/** + * free_image_pages - Free pages allocated for snapshot + */ + +static void free_image_pages(void) +{ + struct pbe * p; + int i; + + p = pagedir_save; + for (i = 0, p = pagedir_save; i < nr_copy_pages; i++, p++) { + if (p->address) { + ClearPageNosave(virt_to_page(p->address)); + free_page(p->address); + p->address = 0; + } + } +} /** * alloc_image_pages - Allocate pages for the snapshot. @@ -701,18 +682,19 @@ static int alloc_image_pages(void) for (i = 0, p = pagedir_save; i < nr_copy_pages; i++, p++) { p->address = get_zeroed_page(GFP_ATOMIC | __GFP_COLD); - if(!p->address) - goto Error; + if (!p->address) + return -ENOMEM; SetPageNosave(virt_to_page(p->address)); } return 0; - Error: - do { - if (p->address) - free_page(p->address); - p->address = 0; - } while (p-- > pagedir_save); - return -ENOMEM; +} + +void swsusp_free(void) +{ + BUG_ON(PageNosave(virt_to_page(pagedir_save))); + BUG_ON(PageNosaveFree(virt_to_page(pagedir_save))); + free_image_pages(); + free_pages((unsigned long) pagedir_save, pagedir_order); } @@ -1105,6 +1087,7 @@ static int __init check_header(void) return -EPERM; } nr_copy_pages = swsusp_info.image_pages; + pagedir_order = get_bitmask_order(SUSPEND_PD_PAGES(nr_copy_pages)); return error; } @@ -1171,9 +1154,7 @@ static int __init read_pagedir(void) int i, n = swsusp_info.pagedir_pages; int error = 0; - pagedir_order = get_bitmask_order(n); - - addr =__get_free_pages(GFP_ATOMIC, pagedir_order); + addr = __get_free_pages(GFP_ATOMIC, pagedir_order); if (!addr) return -ENOMEM; pagedir_nosave = (struct pbe *)addr; diff -puN mm/page_alloc.c~swsusp-kill-on2-algorithm-in-swsusp mm/page_alloc.c --- 25/mm/page_alloc.c~swsusp-kill-on2-algorithm-in-swsusp 2005-01-01 21:25:56.116877056 -0800 +++ 25-akpm/mm/page_alloc.c 2005-01-01 21:25:56.125875688 -0800 @@ -452,26 +452,30 @@ static void __drain_pages(unsigned int c #endif /* CONFIG_PM || CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_PM -int is_head_of_free_region(struct page *page) + +void mark_free_pages(struct zone *zone) { - struct zone *zone = page_zone(page); - unsigned long flags; + unsigned long zone_pfn, flags; int order; struct list_head *curr; - /* - * Should not matter as we need quiescent system for - * suspend anyway, but... - */ + if (!zone->spanned_pages) + return; + spin_lock_irqsave(&zone->lock, flags); + for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) + ClearPageNosaveFree(pfn_to_page(zone_pfn + zone->zone_start_pfn)); + for (order = MAX_ORDER - 1; order >= 0; --order) - list_for_each(curr, &zone->free_area[order].free_list) - if (page == list_entry(curr, struct page, lru)) { - spin_unlock_irqrestore(&zone->lock, flags); - return 1 << order; - } + list_for_each(curr, &zone->free_area[order].free_list) { + unsigned long start_pfn, i; + + start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); + + for (i=0; i < (1<lock, flags); - return 0; } /* _