From: William Lee Irwin III Eliminate specialized page and bh waitqueue hashing structures in favor of a standardized structure, using wake_up_bit() to wake waiters using the standardized wait_bit_key structure. Signed-off-by: Andrew Morton --- 25-akpm/fs/buffer.c | 52 +++++------------------------------ 25-akpm/include/linux/wait.h | 26 +++++++++++++++++ 25-akpm/kernel/wait.c | 23 +++++++++++++++ 25-akpm/mm/filemap.c | 63 ++++++------------------------------------- 4 files changed, 66 insertions(+), 98 deletions(-) diff -puN fs/buffer.c~standardize-bit-waiting-data-type fs/buffer.c --- 25/fs/buffer.c~standardize-bit-waiting-data-type 2004-09-30 22:36:43.479261752 -0700 +++ 25-akpm/fs/buffer.c 2004-09-30 22:36:43.490260080 -0700 @@ -45,26 +45,6 @@ static void invalidate_bh_lrus(void); #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) -struct bh_wait_queue { - struct buffer_head *bh; - wait_queue_t wait; -}; - -#define __DEFINE_BH_WAIT(name, b, f) \ - struct bh_wait_queue name = { \ - .bh = b, \ - .wait = { \ - .task = current, \ - .flags = f, \ - .func = bh_wake_function, \ - .task_list = \ - LIST_HEAD_INIT(name.wait.task_list),\ - }, \ - } -#define DEFINE_BH_WAIT(name, bh) __DEFINE_BH_WAIT(name, bh, 0) -#define DEFINE_BH_WAIT_EXCLUSIVE(name, bh) \ - __DEFINE_BH_WAIT(name, bh, WQ_FLAG_EXCLUSIVE) - /* * Hashed waitqueue_head's for wait_on_buffer() */ @@ -95,24 +75,10 @@ void wake_up_buffer(struct buffer_head * wait_queue_head_t *wq = bh_waitq_head(bh); smp_mb(); - if (waitqueue_active(wq)) - __wake_up(wq, TASK_INTERRUPTIBLE|TASK_UNINTERRUPTIBLE, 1, bh); + __wake_up_bit(wq, &bh->b_state, BH_Lock); } EXPORT_SYMBOL(wake_up_buffer); -static int bh_wake_function(wait_queue_t *wait, unsigned mode, - int sync, void *key) -{ - struct buffer_head *bh = key; - struct bh_wait_queue *wq; - - wq = container_of(wait, struct bh_wait_queue, wait); - if (wq->bh != bh || buffer_locked(bh)) - return 0; - else - return autoremove_wake_function(wait, mode, sync, key); -} - static void sync_buffer(struct buffer_head *bh) { struct block_device *bd; @@ -126,7 +92,7 @@ static void sync_buffer(struct buffer_he void fastcall __lock_buffer(struct buffer_head *bh) { wait_queue_head_t *wqh = bh_waitq_head(bh); - DEFINE_BH_WAIT_EXCLUSIVE(wait, bh); + DEFINE_WAIT_BIT(wait, &bh->b_state, BH_Lock); do { prepare_to_wait_exclusive(wqh, &wait.wait, @@ -155,15 +121,13 @@ void fastcall unlock_buffer(struct buffe void __wait_on_buffer(struct buffer_head * bh) { wait_queue_head_t *wqh = bh_waitq_head(bh); - DEFINE_BH_WAIT(wait, bh); + DEFINE_WAIT_BIT(wait, &bh->b_state, BH_Lock); - do { - prepare_to_wait(wqh, &wait.wait, TASK_UNINTERRUPTIBLE); - if (buffer_locked(bh)) { - sync_buffer(bh); - io_schedule(); - } - } while (buffer_locked(bh)); + prepare_to_wait(wqh, &wait.wait, TASK_UNINTERRUPTIBLE); + if (buffer_locked(bh)) { + sync_buffer(bh); + io_schedule(); + } finish_wait(wqh, &wait.wait); } diff -puN include/linux/wait.h~standardize-bit-waiting-data-type include/linux/wait.h --- 25/include/linux/wait.h~standardize-bit-waiting-data-type 2004-09-30 22:36:43.480261600 -0700 +++ 25-akpm/include/linux/wait.h 2004-09-30 22:36:43.491259928 -0700 @@ -37,6 +37,16 @@ struct __wait_queue { struct list_head task_list; }; +struct wait_bit_key { + void *flags; + int bit_nr; +}; + +struct wait_bit_queue { + struct wait_bit_key key; + wait_queue_t wait; +}; + struct __wait_queue_head { spinlock_t lock; struct list_head task_list; @@ -63,6 +73,9 @@ typedef struct __wait_queue_head wait_qu #define DECLARE_WAIT_QUEUE_HEAD(name) \ wait_queue_head_t name = __WAIT_QUEUE_HEAD_INITIALIZER(name) +#define __WAIT_BIT_KEY_INITIALIZER(word, bit) \ + { .flags = word, .bit_nr = bit, } + static inline void init_waitqueue_head(wait_queue_head_t *q) { q->lock = SPIN_LOCK_UNLOCKED; @@ -125,6 +138,7 @@ static inline void __remove_wait_queue(w void FASTCALL(__wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key)); extern void FASTCALL(__wake_up_locked(wait_queue_head_t *q, unsigned int mode)); extern void FASTCALL(__wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr)); +void FASTCALL(__wake_up_bit(wait_queue_head_t *, void *, int)); #define wake_up(x) __wake_up(x, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1, NULL) #define wake_up_nr(x, nr) __wake_up(x, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, nr, NULL) @@ -300,6 +314,7 @@ void FASTCALL(prepare_to_wait_exclusive( wait_queue_t *wait, int state)); void FASTCALL(finish_wait(wait_queue_head_t *q, wait_queue_t *wait)); int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key); +int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key); #define DEFINE_WAIT(name) \ wait_queue_t name = { \ @@ -310,6 +325,17 @@ int autoremove_wake_function(wait_queue_ }, \ } +#define DEFINE_WAIT_BIT(name, word, bit) \ + struct wait_bit_queue name = { \ + .key = __WAIT_BIT_KEY_INITIALIZER(word, bit), \ + .wait = { \ + .task = current, \ + .func = wake_bit_function, \ + .task_list = \ + LIST_HEAD_INIT(name.wait.task_list), \ + }, \ + } + #define init_wait(wait) \ do { \ wait->task = current; \ diff -puN kernel/wait.c~standardize-bit-waiting-data-type kernel/wait.c --- 25/kernel/wait.c~standardize-bit-waiting-data-type 2004-09-30 22:36:43.482261296 -0700 +++ 25-akpm/kernel/wait.c 2004-09-30 22:36:43.492259776 -0700 @@ -127,3 +127,26 @@ int autoremove_wake_function(wait_queue_ return ret; } EXPORT_SYMBOL(autoremove_wake_function); + +int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) +{ + struct wait_bit_key *key = arg; + struct wait_bit_queue *wait_bit + = container_of(wait, struct wait_bit_queue, wait); + + if (wait_bit->key.flags != key->flags || + wait_bit->key.bit_nr != key->bit_nr || + test_bit(key->bit_nr, key->flags)) + return 0; + else + return autoremove_wake_function(wait, mode, sync, key); +} +EXPORT_SYMBOL(wake_bit_function); + +void fastcall __wake_up_bit(wait_queue_head_t *wq, void *word, int bit) +{ + struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); + if (waitqueue_active(wq)) + __wake_up(wq, TASK_INTERRUPTIBLE|TASK_UNINTERRUPTIBLE, 1, &key); +} +EXPORT_SYMBOL(__wake_up_bit); diff -puN mm/filemap.c~standardize-bit-waiting-data-type mm/filemap.c --- 25/mm/filemap.c~standardize-bit-waiting-data-type 2004-09-30 22:36:43.484260992 -0700 +++ 25-akpm/mm/filemap.c 2004-09-30 22:36:43.494259472 -0700 @@ -363,40 +363,6 @@ EXPORT_SYMBOL(add_to_page_cache_lru); * at a cost of "thundering herd" phenomena during rare hash * collisions. */ -struct page_wait_queue { - struct page *page; - int bit; - wait_queue_t wait; -}; - -static int page_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key) -{ - struct page *page = key; - struct page_wait_queue *wq; - - wq = container_of(wait, struct page_wait_queue, wait); - if (wq->page != page || test_bit(wq->bit, &page->flags)) - return 0; - else - return autoremove_wake_function(wait, mode, sync, NULL); -} - -#define __DEFINE_PAGE_WAIT(name, p, b, f) \ - struct page_wait_queue name = { \ - .page = p, \ - .bit = b, \ - .wait = { \ - .task = current, \ - .func = page_wake_function, \ - .flags = f, \ - .task_list = LIST_HEAD_INIT(name.wait.task_list),\ - }, \ - } - -#define DEFINE_PAGE_WAIT(name, p, b) __DEFINE_PAGE_WAIT(name, p, b, 0) -#define DEFINE_PAGE_WAIT_EXCLUSIVE(name, p, b) \ - __DEFINE_PAGE_WAIT(name, p, b, WQ_FLAG_EXCLUSIVE) - static wait_queue_head_t *page_waitqueue(struct page *page) { const struct zone *zone = page_zone(page); @@ -404,27 +370,16 @@ static wait_queue_head_t *page_waitqueue return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; } -static void wake_up_page(struct page *page) -{ - const unsigned int mode = TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE; - wait_queue_head_t *waitqueue = page_waitqueue(page); - - if (waitqueue_active(waitqueue)) - __wake_up(waitqueue, mode, 1, page); -} - void fastcall wait_on_page_bit(struct page *page, int bit_nr) { wait_queue_head_t *waitqueue = page_waitqueue(page); - DEFINE_PAGE_WAIT(wait, page, bit_nr); + DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); - do { - prepare_to_wait(waitqueue, &wait.wait, TASK_UNINTERRUPTIBLE); - if (test_bit(bit_nr, &page->flags)) { - sync_page(page); - io_schedule(); - } - } while (test_bit(bit_nr, &page->flags)); + prepare_to_wait(waitqueue, &wait.wait, TASK_UNINTERRUPTIBLE); + if (test_bit(bit_nr, &page->flags)) { + sync_page(page); + io_schedule(); + } finish_wait(waitqueue, &wait.wait); } @@ -451,7 +406,7 @@ void fastcall unlock_page(struct page *p if (!TestClearPageLocked(page)) BUG(); smp_mb__after_clear_bit(); - wake_up_page(page); + __wake_up_bit(page_waitqueue(page), &page->flags, PG_locked); } EXPORT_SYMBOL(unlock_page); @@ -467,7 +422,7 @@ void end_page_writeback(struct page *pag BUG(); smp_mb__after_clear_bit(); } - wake_up_page(page); + __wake_up_bit(page_waitqueue(page), &page->flags, PG_writeback); } EXPORT_SYMBOL(end_page_writeback); @@ -483,7 +438,7 @@ EXPORT_SYMBOL(end_page_writeback); void fastcall __lock_page(struct page *page) { wait_queue_head_t *wqh = page_waitqueue(page); - DEFINE_PAGE_WAIT_EXCLUSIVE(wait, page, PG_locked); + DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); while (TestSetPageLocked(page)) { prepare_to_wait_exclusive(wqh, &wait.wait, TASK_UNINTERRUPTIBLE); _