diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/alpha/kernel/time.c xx/arch/alpha/kernel/time.c --- xx-ref/arch/alpha/kernel/time.c 2003-05-27 01:43:50.000000000 +0200 +++ xx/arch/alpha/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -48,7 +49,7 @@ #include "proto.h" #include "irq_impl.h" -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; /* kernel/timer.c */ static int set_rtc_mmss(unsigned long); @@ -101,7 +102,7 @@ void timer_interrupt(int irq, void *dev, alpha_do_profile(regs->pc); #endif - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); /* * Calculate how many ticks have passed since the last update, @@ -133,7 +134,7 @@ void timer_interrupt(int irq, void *dev, state.last_rtc_update = xtime.tv_sec - (tmp ? 600 : 0); } - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } void @@ -392,18 +393,19 @@ time_init(void) void do_gettimeofday(struct timeval *tv) { - unsigned long sec, usec, lost, flags; + unsigned long sec, usec, lost, seq; unsigned long delta_cycles, delta_usec, partial_tick; - read_lock_irqsave(&xtime_lock, flags); + do { + seq = fr_read_begin(&xtime_lock); - delta_cycles = rpcc() - state.last_time; - sec = xtime.tv_sec; - usec = xtime.tv_usec; - partial_tick = state.partial_tick; - lost = jiffies - wall_jiffies; + delta_cycles = rpcc() - state.last_time; + sec = xtime.tv_sec; + usec = xtime.tv_usec; + partial_tick = state.partial_tick; + lost = jiffies - wall_jiffies; - read_unlock_irqrestore(&xtime_lock, flags); + } while (seq != fr_read_end(&xtime_lock)); #ifdef CONFIG_SMP /* Until and unless we figure out how to get cpu cycle counters @@ -445,7 +447,7 @@ do_settimeofday(struct timeval *tv) unsigned long delta_usec; long sec, usec; - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* The offset that is added into time in do_gettimeofday above must be subtracted out here to keep a coherent view of the @@ -476,7 +478,7 @@ do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/arm/kernel/time.c xx/arch/arm/kernel/time.c --- xx-ref/arch/arm/kernel/time.c 2003-05-27 01:43:50.000000000 +0200 +++ xx/arch/arm/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,7 @@ #include extern int setup_arm_irq(int, struct irqaction *); -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; /* change this if you have some constant time drift */ @@ -151,20 +152,19 @@ static void do_leds(void) void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long lost, seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; + do { + seq = fr_read_begin(&xtime_lock); + usec = gettimeoffset(); + lost = jiffies - wall_jiffies; if (lost) usec += lost * USECS_PER_JIFFY; - } - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (seq != fr_read_end(&xtime_lock)); /* usec may have gone up a lot: be safe */ while (usec >= 1000000) { @@ -178,7 +178,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -198,7 +198,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } static struct irqaction timer_irq = { diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/i386/kernel/time.c xx/arch/i386/kernel/time.c --- xx-ref/arch/i386/kernel/time.c 2003-05-27 01:43:51.000000000 +0200 +++ xx/arch/i386/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -79,7 +80,7 @@ static unsigned long last_tsc_low; /* ls */ unsigned long fast_gettimeoffset_quotient; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED; @@ -158,6 +159,7 @@ extern spinlock_t i8259A_lock; static unsigned long do_slow_gettimeoffset(void) { int count; + unsigned long flags; static int count_p = LATCH; /* for the first call after boot */ static unsigned long jiffies_p = 0; @@ -168,7 +170,7 @@ static unsigned long do_slow_gettimeoffs unsigned long jiffies_t; /* gets recalled with irq locally disabled */ - spin_lock(&i8253_lock); + spin_lock_irqsave(&i8253_lock, flags); /* timer count may underflow right here */ outb_p(0x00, 0x43); /* latch the count ASAP */ @@ -190,7 +192,7 @@ static unsigned long do_slow_gettimeoffs count = LATCH - 1; } - spin_unlock(&i8253_lock); + spin_unlock_irqrestore(&i8253_lock, flags); /* * avoiding timer inconsistencies (they are rare, but they happen)... @@ -211,13 +213,13 @@ static unsigned long do_slow_gettimeoffs int i; - spin_lock(&i8259A_lock); + spin_lock_irqsave(&i8259A_lock, flags); /* * This is tricky when I/O APICs are used; * see do_timer_interrupt(). */ i = inb(0x20); - spin_unlock(&i8259A_lock); + spin_unlock_irqrestore(&i8259A_lock, flags); /* assumption about timer being IRQ0 */ if (i & 0x01) { @@ -441,19 +443,21 @@ void __cyclone_delay(unsigned long loops */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = fr_read_begin(&xtime_lock); + usec = do_gettimeoffset(); + { + unsigned long lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000 / HZ); + } + + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -466,7 +470,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -486,7 +490,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* @@ -652,7 +656,7 @@ static void timer_interrupt(int irq, voi * the irq version of write_lock because as just said we have irq * locally disabled. -arca */ - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); if(use_cyclone) mark_timeoffset_cyclone(); @@ -708,7 +712,7 @@ static void timer_interrupt(int irq, voi do_timer_interrupt(irq, NULL, regs); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/ia64/kernel/time.c xx/arch/ia64/kernel/time.c --- xx-ref/arch/ia64/kernel/time.c 2002-11-29 02:22:55.000000000 +0100 +++ xx/arch/ia64/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -23,7 +24,7 @@ #include #include -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; extern unsigned long last_time_offset; @@ -86,7 +87,7 @@ gettimeoffset (void) void do_settimeofday (struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); { /* * This is revolting. We need to set "xtime" correctly. However, the value @@ -108,16 +109,16 @@ do_settimeofday (struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; } - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } void do_gettimeofday (struct timeval *tv) { - unsigned long flags, usec, sec, old; + unsigned long seq, usec, sec, old; - read_lock_irqsave(&xtime_lock, flags); - { + do { + seq = fr_read_begin(&xtime_lock); usec = gettimeoffset(); /* @@ -134,8 +135,7 @@ do_gettimeofday (struct timeval *tv) sec = xtime.tv_sec; usec += xtime.tv_usec; - } - read_unlock_irqrestore(&xtime_lock, flags); + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -178,10 +178,10 @@ timer_interrupt(int irq, void *dev_id, s * another CPU. We need to avoid to SMP race by acquiring the * xtime_lock. */ - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); do_timer(regs); local_cpu_data->itm_next = new_itm; - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } else local_cpu_data->itm_next = new_itm; diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/m68k/kernel/time.c xx/arch/m68k/kernel/time.c --- xx-ref/arch/m68k/kernel/time.c 2003-05-27 01:43:51.000000000 +0200 +++ xx/arch/m68k/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -102,7 +103,7 @@ void time_init(void) mach_sched_init(timer_interrupt); } -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; /* * This version of gettimeofday has near microsecond resolution. @@ -110,17 +111,18 @@ extern rwlock_t xtime_lock; void do_gettimeofday(struct timeval *tv) { extern unsigned long wall_jiffies; - unsigned long flags; + unsigned long seq; unsigned long usec, sec, lost; - read_lock_irqsave(&xtime_lock, flags); - usec = mach_gettimeoffset(); - lost = jiffies - wall_jiffies; - if (lost) - usec += lost * (1000000/HZ); - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = fr_read_begin(&xtime_lock); + usec = mach_gettimeoffset(); + lost = jiffies - wall_jiffies; + if (lost) + usec += lost * (1000000/HZ); + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -133,7 +135,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -152,5 +154,5 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/mips/kernel/time.c xx/arch/mips/kernel/time.c --- xx-ref/arch/mips/kernel/time.c 2002-11-29 02:22:56.000000000 +0100 +++ xx/arch/mips/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,7 @@ /* * forward reference */ -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern volatile unsigned long wall_jiffies; /* @@ -65,20 +66,21 @@ int (*rtc_set_time)(unsigned long) = nul */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_gettimeoffset(); + do { + seq = fr_read_begin (&xtime_lock); + *tv = xtime; + tv->tv_usec += do_gettimeoffset(); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + /* + * xtime is atomically updated in timer_bh. jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; - read_unlock_irqrestore (&xtime_lock, flags); + } while (seq != fr_read_end (&xtime_lock)); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -88,7 +90,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + fr_write_lock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is @@ -108,7 +110,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + fr_write_unlock_irq (&xtime_lock); } @@ -360,7 +362,7 @@ void timer_interrupt(int irq, void *dev_ * CMOS clock accordingly every ~11 minutes. rtc_set_time() has to be * called as close as possible to 500 ms before the new second starts. */ - read_lock (&xtime_lock); + fr_read_begin (&xtime_lock); if ((time_status & STA_UNSYNC) == 0 && xtime.tv_sec > last_rtc_update + 660 && xtime.tv_usec >= 500000 - ((unsigned) tick) / 2 && @@ -372,7 +374,7 @@ void timer_interrupt(int irq, void *dev_ /* do it again in 60 s */ } } - read_unlock (&xtime_lock); + fr_read_end (&xtime_lock); /* * If jiffies has overflowed in this timer_interrupt we must diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/mips64/kernel/time.c xx/arch/mips64/kernel/time.c --- xx-ref/arch/mips64/kernel/time.c 2002-11-29 02:22:56.000000000 +0100 +++ xx/arch/mips64/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,7 @@ /* * forward reference */ -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern volatile unsigned long wall_jiffies; /* @@ -65,20 +66,21 @@ int (*rtc_set_time)(unsigned long) = nul */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; - read_lock_irqsave (&xtime_lock, flags); - *tv = xtime; - tv->tv_usec += do_gettimeoffset(); + do { + seq = fr_read_begin (&xtime_lock); + *tv = xtime; + tv->tv_usec += do_gettimeoffset(); - /* - * xtime is atomically updated in timer_bh. jiffies - wall_jiffies - * is nonzero if the timer bottom half hasnt executed yet. - */ - if (jiffies - wall_jiffies) - tv->tv_usec += USECS_PER_JIFFY; + /* + * xtime is atomically updated in timer_bh. jiffies - wall_jiffies + * is nonzero if the timer bottom half hasnt executed yet. + */ + if (jiffies - wall_jiffies) + tv->tv_usec += USECS_PER_JIFFY; - read_unlock_irqrestore (&xtime_lock, flags); + } while (seq != fr_read_end (&xtime_lock)); if (tv->tv_usec >= 1000000) { tv->tv_usec -= 1000000; @@ -88,7 +90,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq (&xtime_lock); + fr_write_lock_irq (&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is @@ -108,7 +110,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq (&xtime_lock); + fr_write_unlock_irq (&xtime_lock); } @@ -360,7 +362,7 @@ void timer_interrupt(int irq, void *dev_ * CMOS clock accordingly every ~11 minutes. rtc_set_time() has to be * called as close as possible to 500 ms before the new second starts. */ - read_lock (&xtime_lock); + fr_read_begin (&xtime_lock); if ((time_status & STA_UNSYNC) == 0 && xtime.tv_sec > last_rtc_update + 660 && xtime.tv_usec >= 500000 - ((unsigned) tick) / 2 && @@ -372,7 +374,7 @@ void timer_interrupt(int irq, void *dev_ /* do it again in 60 s */ } } - read_unlock (&xtime_lock); + fr_read_end (&xtime_lock); /* * If jiffies has overflowed in this timer_interrupt we must diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/parisc/kernel/time.c xx/arch/parisc/kernel/time.c --- xx-ref/arch/parisc/kernel/time.c 2002-11-29 02:22:56.000000000 +0100 +++ xx/arch/parisc/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,7 @@ /* xtime and wall_jiffies keep wall-clock time */ extern unsigned long wall_jiffies; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; static long clocktick; /* timer cycles per tick */ static long halftick; @@ -95,7 +96,7 @@ void timer_interrupt(int irq, void *dev_ #endif if (cpu == 0) { extern int pc_in_user_space; - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); #ifndef CONFIG_SMP if (!user_mode(regs)) parisc_do_profile(regs->iaoq[0]); @@ -103,7 +104,7 @@ void timer_interrupt(int irq, void *dev_ parisc_do_profile(&pc_in_user_space); #endif do_timer(regs); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } } @@ -158,16 +159,15 @@ gettimeoffset (void) void do_gettimeofday (struct timeval *tv) { - unsigned long flags, usec, sec; + unsigned long seq, usec, sec; - read_lock_irqsave(&xtime_lock, flags); - { + do { + seq = fr_read_begin(&xtime_lock); usec = gettimeoffset(); sec = xtime.tv_sec; usec += xtime.tv_usec; - } - read_unlock_irqrestore(&xtime_lock, flags); + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -181,7 +181,7 @@ do_gettimeofday (struct timeval *tv) void do_settimeofday (struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); { /* * This is revolting. We need to set "xtime" @@ -204,7 +204,7 @@ do_settimeofday (struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; } - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } @@ -226,10 +226,10 @@ void __init time_init(void) mtctl(next_tick, 16); if(pdc_tod_read(&tod_data) == 0) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); xtime.tv_sec = tod_data.tod_sec; xtime.tv_usec = tod_data.tod_usec; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } else { printk(KERN_ERR "Error reading tod clock\n"); xtime.tv_sec = 0; diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/ppc/kernel/time.c xx/arch/ppc/kernel/time.c --- xx-ref/arch/ppc/kernel/time.c 2003-05-27 01:43:52.000000000 +0200 +++ xx/arch/ppc/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -56,6 +56,7 @@ #include #include #include +#include #include #include @@ -73,7 +74,7 @@ extern int do_sys_settimeofday(struct ti /* keep track of when we need to update the rtc */ time_t last_rtc_update; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; /* The decrementer counts down by 128 every 128ns on a 601. */ #define DECREMENTER_COUNT_601 (1000000000 / HZ) @@ -163,7 +164,7 @@ int timer_interrupt(struct pt_regs * reg continue; /* We are in an interrupt, no need to save/restore flags */ - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); tb_last_stamp = jiffy_stamp; do_timer(regs); @@ -193,7 +194,7 @@ int timer_interrupt(struct pt_regs * reg /* Try again one minute later */ last_rtc_update += 60; } - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } if (!disarm_decr[cpu]) set_dec(next_dec); @@ -216,22 +217,23 @@ int timer_interrupt(struct pt_regs * reg */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; unsigned delta, lost_ticks, usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = xtime.tv_usec; - delta = tb_ticks_since(tb_last_stamp); + do { + seq = fr_read_begin(&xtime_lock); + sec = xtime.tv_sec; + usec = xtime.tv_usec; + delta = tb_ticks_since(tb_last_stamp); #ifdef CONFIG_SMP - /* As long as timebases are not in sync, gettimeofday can only - * have jiffy resolution on SMP. - */ - if (!smp_tb_synchronized) - delta = 0; + /* As long as timebases are not in sync, gettimeofday can only + * have jiffy resolution on SMP. + */ + if (!smp_tb_synchronized) + delta = 0; #endif /* CONFIG_SMP */ - lost_ticks = jiffies - wall_jiffies; - read_unlock_irqrestore(&xtime_lock, flags); + lost_ticks = jiffies - wall_jiffies; + } while (seq != fr_read_end(&xtime_lock)); usec += mulhwu(tb_to_us, tb_ticks_per_jiffy * lost_ticks + delta); while (usec >= 1000000) { @@ -247,7 +249,7 @@ void do_settimeofday(struct timeval *tv) unsigned long flags; int tb_delta, new_usec, new_sec; - write_lock_irqsave(&xtime_lock, flags); + fr_write_lock_irqsave(&xtime_lock, flags); /* Updating the RTC is not the job of this code. If the time is * stepped under NTP, the RTC will be update after STA_UNSYNC * is cleared. Tool like clock/hwclock either copy the RTC @@ -287,7 +289,7 @@ void do_settimeofday(struct timeval *tv) time_state = TIME_ERROR; /* p. 24, (a) */ time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irqrestore(&xtime_lock, flags); + fr_write_unlock_irqrestore(&xtime_lock, flags); } /* This function is only called on the boot processor */ diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/ppc64/kernel/time.c xx/arch/ppc64/kernel/time.c --- xx-ref/arch/ppc64/kernel/time.c 2003-05-27 01:43:53.000000000 +0200 +++ xx/arch/ppc64/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -69,7 +70,7 @@ extern void setup_before_console_init(); /* keep track of when we need to update the rtc */ time_t last_rtc_update; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern int piranha_simulator; #ifdef CONFIG_PPC_ISERIES unsigned long iSeries_recal_titan = 0; @@ -268,12 +269,12 @@ int timer_interrupt(struct pt_regs * reg smp_local_timer_interrupt(regs); #endif if (cpu == 0) { - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); tb_last_stamp = lpaca->next_jiffy_update_tb; do_timer(regs); timer_sync_xtime( cur_tb ); timer_check_rtc(); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); if ( adjusting_time && (time_adjust == 0) ) ppc_adjtimex(); } @@ -378,7 +379,7 @@ void do_settimeofday(struct timeval *tv) long int tb_delta, new_usec, new_sec; unsigned long new_xsec; - write_lock_irqsave(&xtime_lock, flags); + fr_write_lock_irqsave(&xtime_lock, flags); /* Updating the RTC is not the job of this code. If the time is * stepped under NTP, the RTC will be update after STA_UNSYNC * is cleared. Tool like clock/hwclock either copy the RTC @@ -432,7 +433,7 @@ void do_settimeofday(struct timeval *tv) systemcfg->tz_minuteswest = sys_tz.tz_minuteswest; systemcfg->tz_dsttime = sys_tz.tz_dsttime; - write_unlock_irqrestore(&xtime_lock, flags); + fr_write_unlock_irqrestore(&xtime_lock, flags); } /* @@ -496,7 +497,7 @@ void __init time_init(void) if ( ! piranha_simulator ) { ppc_md.get_boot_time(&tm); } - write_lock_irqsave(&xtime_lock, flags); + fr_write_lock_irqsave(&xtime_lock, flags); xtime.tv_sec = mktime(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); tb_last_stamp = get_tb(); @@ -513,7 +514,7 @@ void __init time_init(void) xtime.tv_usec = 0; last_rtc_update = xtime.tv_sec; - write_unlock_irqrestore(&xtime_lock, flags); + fr_write_unlock_irqrestore(&xtime_lock, flags); /* Not exact, but the timer interrupt takes care of this */ set_dec(tb_ticks_per_jiffy); @@ -621,7 +622,7 @@ void ppc_adjtimex(void) new_tb_to_xs = divres.result_low; new_xsec = mulhdu( tb_ticks, new_tb_to_xs ); - write_lock_irqsave( &xtime_lock, flags ); + fr_write_lock_irqsave( &xtime_lock, flags ); old_xsec = mulhdu( tb_ticks, systemcfg->tb_to_xs ); new_stamp_xsec = systemcfg->stamp_xsec + old_xsec - new_xsec; @@ -641,7 +642,7 @@ void ppc_adjtimex(void) wmb(); ++(systemcfg->tb_update_count); - write_unlock_irqrestore( &xtime_lock, flags ); + fr_write_unlock_irqrestore( &xtime_lock, flags ); } diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/s390/kernel/time.c xx/arch/s390/kernel/time.c --- xx-ref/arch/s390/kernel/time.c 2003-05-27 01:43:53.000000000 +0200 +++ xx/arch/s390/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -42,7 +43,7 @@ static ext_int_info_t ext_int_info_timer; static uint64_t init_timer_cc; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; void tod_to_timeval(__u64 todval, struct timeval *xtime) @@ -95,13 +96,14 @@ static inline unsigned long do_gettimeof */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = xtime.tv_usec + do_gettimeoffset(); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = fr_read_begin(&xtime_lock); + sec = xtime.tv_sec; + usec = xtime.tv_usec + do_gettimeoffset(); + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -115,7 +117,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -134,7 +136,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* @@ -160,13 +162,13 @@ static void do_comparator_interrupt(stru #ifdef CONFIG_SMP if (S390_lowcore.cpu_data.cpu_addr == boot_cpu_addr) - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); update_process_times(user_mode(regs)); if (S390_lowcore.cpu_data.cpu_addr == boot_cpu_addr) { do_timer(regs); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } #else do_timer(regs); diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/s390x/kernel/time.c xx/arch/s390x/kernel/time.c --- xx-ref/arch/s390x/kernel/time.c 2003-05-27 01:43:53.000000000 +0200 +++ xx/arch/s390x/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -42,7 +43,7 @@ static ext_int_info_t ext_int_info_timer; static uint64_t init_timer_cc; -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; void tod_to_timeval(__u64 todval, struct timeval *xtime) @@ -68,13 +69,14 @@ static inline unsigned long do_gettimeof */ void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - sec = xtime.tv_sec; - usec = xtime.tv_usec + do_gettimeoffset(); - read_unlock_irqrestore(&xtime_lock, flags); + do { + seq = fr_read_begin(&xtime_lock); + sec = xtime.tv_sec; + usec = xtime.tv_usec + do_gettimeoffset(); + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -88,7 +90,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* This is revolting. We need to set the xtime.tv_usec * correctly. However, the value in this location is * is value at the last tick. @@ -107,7 +109,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* @@ -133,13 +135,13 @@ static void do_comparator_interrupt(stru #ifdef CONFIG_SMP if (S390_lowcore.cpu_data.cpu_addr == boot_cpu_addr) - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); update_process_times(user_mode(regs)); if (S390_lowcore.cpu_data.cpu_addr == boot_cpu_addr) { do_timer(regs); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } #else do_timer(regs); diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/sh/kernel/time.c xx/arch/sh/kernel/time.c --- xx-ref/arch/sh/kernel/time.c 2002-01-22 18:53:20.000000000 +0100 +++ xx/arch/sh/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -70,7 +71,7 @@ #endif /* CONFIG_CPU_SUBTYPE_ST40STB1 */ #endif /* __sh3__ or __SH4__ */ -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; #define TICK_SIZE tick @@ -125,19 +126,18 @@ static unsigned long do_gettimeoffset(vo void do_gettimeofday(struct timeval *tv) { - unsigned long flags; + unsigned long seq, lost; unsigned long usec, sec; - read_lock_irqsave(&xtime_lock, flags); - usec = do_gettimeoffset(); - { - unsigned long lost = jiffies - wall_jiffies; + do { + seq = fr_read_begin(&xtime_lock); + usec = do_gettimeoffset(); + lost = jiffies - wall_jiffies; if (lost) usec += lost * (1000000 / HZ); - } - sec = xtime.tv_sec; - usec += xtime.tv_usec; - read_unlock_irqrestore(&xtime_lock, flags); + sec = xtime.tv_sec; + usec += xtime.tv_usec; + } while (seq != fr_read_end(&xtime_lock)); while (usec >= 1000000) { usec -= 1000000; @@ -150,7 +150,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); /* * This is revolting. We need to set "xtime" correctly. However, the * value in this location is the value at the most recent update of @@ -170,7 +170,7 @@ void do_settimeofday(struct timeval *tv) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* last time the RTC clock got updated */ @@ -229,9 +229,9 @@ static void timer_interrupt(int irq, voi * the irq version of write_lock because as just said we have irq * locally disabled. -arca */ - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); do_timer_interrupt(irq, NULL, regs); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } static unsigned int __init get_timer_frequency(void) diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/sparc/kernel/time.c xx/arch/sparc/kernel/time.c --- xx-ref/arch/sparc/kernel/time.c 2003-05-27 01:43:53.000000000 +0200 +++ xx/arch/sparc/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -41,7 +42,7 @@ #include #include -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; enum sparc_clock_type sp_clock_typ; spinlock_t mostek_lock = SPIN_LOCK_UNLOCKED; @@ -133,7 +134,7 @@ void timer_interrupt(int irq, void *dev_ #endif clear_clock_irq(); - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); do_timer(regs); @@ -147,7 +148,7 @@ void timer_interrupt(int irq, void *dev_ else last_rtc_update = xtime.tv_sec - 600; /* do it again in 60 s */ } - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } /* Kick start a stopped clock (procedure from the Sun NVRAM/hostid FAQ). */ @@ -511,9 +512,9 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); bus_do_settimeofday(tv); - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } static void sbus_do_settimeofday(struct timeval *tv) diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/sparc64/kernel/time.c xx/arch/sparc64/kernel/time.c --- xx-ref/arch/sparc64/kernel/time.c 2003-05-27 01:43:54.000000000 +0200 +++ xx/arch/sparc64/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -37,7 +38,7 @@ #include #include -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; extern unsigned long wall_jiffies; spinlock_t mostek_lock = SPIN_LOCK_UNLOCKED; @@ -461,7 +462,7 @@ static void timer_interrupt(int irq, voi { unsigned long ticks, pstate; - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); do { #ifndef CONFIG_SMP @@ -489,13 +490,13 @@ static void timer_interrupt(int irq, voi timer_check_rtc(); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } #ifdef CONFIG_SMP void timer_tick_interrupt(struct pt_regs *regs) { - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); do_timer(regs); @@ -506,7 +507,7 @@ void timer_tick_interrupt(struct pt_regs timer_check_rtc(); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } #endif @@ -1004,7 +1005,7 @@ void do_settimeofday(struct timeval *tv) if (this_is_starfire) return; - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); tv->tv_usec -= do_gettimeoffset(); tv->tv_usec -= (jiffies - wall_jiffies) * (1000000 / HZ); @@ -1020,7 +1021,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } void do_gettimeofday(struct timeval *tv) diff -urNp --exclude CVS --exclude BitKeeper xx-ref/arch/x86_64/kernel/time.c xx/arch/x86_64/kernel/time.c --- xx-ref/arch/x86_64/kernel/time.c 2003-05-27 01:43:54.000000000 +0200 +++ xx/arch/x86_64/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -19,10 +19,11 @@ #include #include #include +#include #include #include -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; spinlock_t rtc_lock = SPIN_LOCK_UNLOCKED; spinlock_t i8253_lock = SPIN_LOCK_UNLOCKED; @@ -100,7 +101,7 @@ void do_gettimeofday(struct timeval *tv) void do_settimeofday(struct timeval *tv) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); vxtime_lock(); tv->tv_usec -= (jiffies - wall_jiffies) * tick @@ -119,7 +120,7 @@ void do_settimeofday(struct timeval *tv) time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* @@ -202,7 +203,7 @@ static void timer_interrupt(int irq, voi * variables, because both do_timer() and us change them -arca+vojtech */ - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); vxtime_lock(); { @@ -278,7 +279,7 @@ static void timer_interrupt(int irq, voi } vxtime_unlock(); - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); } static unsigned long get_cmos_time(void) @@ -505,10 +506,10 @@ void __init time_init(void) hpet_address = 0; #endif - write_lock(&xtime_lock); + fr_write_lock(&xtime_lock); xtime.tv_sec = get_cmos_time(); xtime.tv_usec = 0; - write_unlock(&xtime_lock); + fr_write_unlock(&xtime_lock); if (!hpet_init()) { vxtime_hz = (1000000000000000L + hpet_period / 2) / hpet_period; diff -urNp --exclude CVS --exclude BitKeeper xx-ref/include/linux/frlock.h xx/include/linux/frlock.h --- xx-ref/include/linux/frlock.h 1970-01-01 01:00:00.000000000 +0100 +++ xx/include/linux/frlock.h 2003-05-27 04:53:23.000000000 +0200 @@ -0,0 +1,101 @@ +#ifndef __LINUX_FRLOCK_H +#define __LINUX_FRLOCK_H + +/* + * Fast read-write spinlocks. + * + * Fast reader/writer locks without starving writers. This type of + * lock for data where the reader wants a consitent set of information + * and is willing to retry if the information changes. Readers never + * block but they may have to retry if a writer is in + * progress. Writers do not wait for readers. + * + * Generalization on sequence variables used for gettimeofday on x86-64 + * by Andrea Arcangeli + * + * This is not as cache friendly as brlock. Also, this will not work + * for data that contains pointers, because any writer could + * invalidate a pointer that a reader was following. + * + * + * Expected reader usage: + * do { + * seq = fr_read_begin(); + * ... + * } while (seq != fr_read_end()); + * + * On non-SMP the spin locks disappear but the writer still needs + * to increment the sequence variables because an interrupt routine could + * change the state of the data. + */ + +#include +#include +#include + +typedef struct { + spinlock_t lock; + unsigned pre_sequence; + unsigned post_sequence; +} frlock_t; + +#define FR_LOCK_UNLOCKED { SPIN_LOCK_UNLOCKED, 0, 0 } +#define frlock_init(x) do { *(x) = (frlock_t) FR_LOCK_UNLOCKED; } while (0) + +static inline void fr_write_lock(frlock_t *rw) +{ + spin_lock(&rw->lock); + rw->pre_sequence++; + wmb(); +} + +static inline void fr_write_unlock(frlock_t *rw) +{ + wmb(); + rw->post_sequence++; + spin_unlock(&rw->lock); +} + +static inline int fr_write_trylock(frlock_t *rw) +{ + int ret = spin_trylock(&rw->lock); + + if (ret) { + ++rw->pre_sequence; + wmb(); + } + return ret; +} + +static inline unsigned fr_read_begin(frlock_t *rw) +{ + unsigned ret = rw->post_sequence; + rmb(); + return ret; + +} + +static inline unsigned fr_read_end(frlock_t *rw) +{ + rmb(); + return rw->pre_sequence; +} + +/* + * Possible sw/hw IRQ protected versions of the interfaces. + */ +#define fr_write_lock_irqsave(lock, flags) \ + do { local_irq_save(flags); fr_write_lock(lock); } while (0) +#define fr_write_lock_irq(lock) \ + do { local_irq_disable(); fr_write_lock(lock); } while (0) +#define fr_write_lock_bh(lock) \ + do { local_bh_disable(); fr_write_lock(lock); } while (0) + +#define fr_write_unlock_irqrestore(lock, flags) \ + do { fr_write_unlock(lock); local_irq_restore(flags); } while(0) +#define fr_write_unlock_irq(lock) \ + do { fr_write_unlock(lock); local_irq_enable(); } while(0) +#define fr_write_unlock_bh(lock) \ + do { fr_write_unlock(lock); local_bh_enable(); } while(0) + +#endif /* __LINUX_FRLOCK_H */ diff -urNp --exclude CVS --exclude BitKeeper xx-ref/kernel/time.c xx/kernel/time.c --- xx-ref/kernel/time.c 2002-11-29 02:23:18.000000000 +0100 +++ xx/kernel/time.c 2003-05-27 04:53:12.000000000 +0200 @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -38,7 +39,7 @@ struct timezone sys_tz; /* The xtime_lock is not only serializing the xtime read/writes but it's also serializing all accesses to the global NTP variables now. */ -extern rwlock_t xtime_lock; +extern frlock_t xtime_lock; #if !defined(__alpha__) && !defined(__ia64__) @@ -79,7 +80,7 @@ asmlinkage long sys_stime(int * tptr) return -EPERM; if (get_user(value, tptr)) return -EFAULT; - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); vxtime_lock(); xtime.tv_sec = value; xtime.tv_usec = 0; @@ -88,7 +89,7 @@ asmlinkage long sys_stime(int * tptr) time_status |= STA_UNSYNC; time_maxerror = NTP_PHASE_LIMIT; time_esterror = NTP_PHASE_LIMIT; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); return 0; } @@ -127,11 +128,11 @@ asmlinkage long sys_gettimeofday(struct */ inline static void warp_clock(void) { - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); vxtime_lock(); xtime.tv_sec += sys_tz.tz_minuteswest * 60; vxtime_unlock(); - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); } /* @@ -235,7 +236,7 @@ int do_adjtimex(struct timex *txc) if (txc->tick < 900000/HZ || txc->tick > 1100000/HZ) return -EINVAL; - write_lock_irq(&xtime_lock); + fr_write_lock_irq(&xtime_lock); result = time_state; /* mostly `TIME_OK' */ /* Save for later - semantics of adjtime is to return old value */ @@ -390,7 +391,7 @@ leave: if ((time_status & (STA_UNSYNC|ST txc->calcnt = pps_calcnt; txc->errcnt = pps_errcnt; txc->stbcnt = pps_stbcnt; - write_unlock_irq(&xtime_lock); + fr_write_unlock_irq(&xtime_lock); do_gettimeofday(&txc->time); return(result); } diff -urNp --exclude CVS --exclude BitKeeper xx-ref/kernel/timer.c xx/kernel/timer.c --- xx-ref/kernel/timer.c 2003-05-27 04:52:56.000000000 +0200 +++ xx/kernel/timer.c 2003-05-27 04:53:12.000000000 +0200 @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -86,7 +87,7 @@ unsigned long wall_jiffies; /* * This spinlock protect us from races in SMP while playing with xtime. -arca */ -rwlock_t xtime_lock = RW_LOCK_UNLOCKED; +frlock_t xtime_lock = FR_LOCK_UNLOCKED; /* * This is the 'global' timer BH. This gets called only if one of