diff -urNp --exclude CVS --exclude BitKeeper x-ref/arch/i386/kernel/io_apic.c x/arch/i386/kernel/io_apic.c --- x-ref/arch/i386/kernel/io_apic.c 2003-05-27 01:43:51.000000000 +0200 +++ x/arch/i386/kernel/io_apic.c 2003-05-27 03:00:32.000000000 +0200 @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -188,6 +189,99 @@ static void clear_IO_APIC (void) clear_IO_APIC_pin(apic, pin); } +static void set_ioapic_affinity (unsigned int irq, unsigned long mask) +{ + unsigned long flags; + + /* + * Only the first 8 bits are valid. + */ + mask = mask << 24; + spin_lock_irqsave(&ioapic_lock, flags); + __DO_ACTION(1, = mask, ) + spin_unlock_irqrestore(&ioapic_lock, flags); +} + +#ifdef CONFIG_SMP + +#define IRQ_BALANCE_INTERVAL (HZ/50) + +typedef struct { + unsigned int cpu; + unsigned long timestamp; +} ____cacheline_aligned irq_balance_t; + +static irq_balance_t irq_balance[NR_IRQS] __cacheline_aligned; +extern unsigned long irq_affinity [NR_IRQS]; + +#define IRQ_ALLOWED(cpu,allowed_mask) \ + ((1UL << cpu) & (allowed_mask)) + +#define ksoftirqd_is_running(phys_id) (cpu_curr(phys_id) == ksoftirqd_task(phys_id)) +#define __irq_idle_cpu(phys_id) (idle_cpu(phys_id) || ksoftirqd_is_running(phys_id)) +#define irq_idle_cpu(phys_id) (__irq_idle_cpu(phys_id) && \ + (smp_num_siblings <= 1 || __irq_idle_cpu(cpu_sibling_map[phys_id]))) + +static unsigned long move(unsigned int curr_cpu, unsigned long allowed_mask, unsigned long now, int direction) +{ + unsigned int cpu = curr_cpu; + unsigned int phys_id; + + phys_id = cpu_logical_map(cpu); + if (IRQ_ALLOWED(phys_id, allowed_mask) && irq_idle_cpu(phys_id)) + return cpu; + + goto inside; + + do { + if (unlikely(cpu == curr_cpu)) + return cpu; +inside: + if (direction == 1) { + cpu++; + if (cpu >= smp_num_cpus) + cpu = 0; + } else { + cpu--; + if (cpu == -1) + cpu = smp_num_cpus-1; + } + + phys_id = cpu_logical_map(cpu); + } while (!IRQ_ALLOWED(phys_id, allowed_mask) || !irq_idle_cpu(phys_id)); + + return cpu; +} + +#endif /* CONFIG_SMP */ + +static inline void balance_irq(int irq) +{ +#if CONFIG_SMP + irq_balance_t *entry = irq_balance + irq; + unsigned long now = jiffies; + + if (unlikely(time_after(now, entry->timestamp + IRQ_BALANCE_INTERVAL))) { + unsigned long allowed_mask; + unsigned int new_cpu; + int random_number; + + entry->timestamp = now; + + rdtscl(random_number); + random_number &= 1; + + allowed_mask = cpu_online_map & irq_affinity[irq]; + new_cpu = move(entry->cpu, allowed_mask, now, random_number); + if (entry->cpu != new_cpu) { + entry->cpu = new_cpu; + set_ioapic_affinity(irq, clustered_apic_mode == 0 ? + 1UL << new_cpu : cpu_present_to_apicid(entry->cpu) ); + } + } +#endif +} + /* * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to * specific CPU-side IRQs. @@ -693,8 +787,7 @@ void __init setup_IO_APIC_irqs(void) } /* - * Set up the 8259A-master output pin as broadcast to all - * CPUs. + * Set up the 8259A-master output pin: */ void __init setup_ExtINT_IRQ0_pin(unsigned int pin, int vector) { @@ -1221,6 +1314,7 @@ static unsigned int startup_edge_ioapic_ */ static void ack_edge_ioapic_irq(unsigned int irq) { + balance_irq(irq); if ((irq_desc[irq].status & (IRQ_PENDING | IRQ_DISABLED)) == (IRQ_PENDING | IRQ_DISABLED)) mask_IO_APIC_irq(irq); @@ -1260,6 +1354,7 @@ static void end_level_ioapic_irq (unsign unsigned long v; int i; + balance_irq(irq); /* * It appears there is an erratum which affects at least version 0x11 * of I/O APIC (that's the 82093AA and cores integrated into various @@ -1316,19 +1411,6 @@ static void end_level_ioapic_irq (unsign static void mask_and_ack_level_ioapic_irq (unsigned int irq) { /* nothing */ } -static void set_ioapic_affinity (unsigned int irq, unsigned long mask) -{ - unsigned long flags; - /* - * Only the first 8 bits are valid. - */ - mask = mask << 24; - - spin_lock_irqsave(&ioapic_lock, flags); - __DO_ACTION(1, = mask, ) - spin_unlock_irqrestore(&ioapic_lock, flags); -} - /* * Level and edge triggered IO-APIC interrupts need different handling, * so we use two separate IRQ descriptors. Edge triggered IRQs can be diff -urNp --exclude CVS --exclude BitKeeper x-ref/arch/i386/kernel/irq.c x/arch/i386/kernel/irq.c --- x-ref/arch/i386/kernel/irq.c 2003-05-27 02:45:29.000000000 +0200 +++ x/arch/i386/kernel/irq.c 2003-05-27 02:45:34.000000000 +0200 @@ -1092,7 +1092,7 @@ out: static struct proc_dir_entry * smp_affinity_entry [NR_IRQS]; -static unsigned long irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = ~0UL }; +unsigned long irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = ~0UL }; static int irq_affinity_read_proc (char *page, char **start, off_t off, int count, int *eof, void *data) {