diff -urNp --exclude CVS --exclude BitKeeper x-ref/arch/i386/kernel/io_apic.c x/arch/i386/kernel/io_apic.c --- x-ref/arch/i386/kernel/io_apic.c 2003-09-22 08:04:57.000000000 +0200 +++ x/arch/i386/kernel/io_apic.c 2003-09-23 03:17:05.000000000 +0200 @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -198,6 +199,106 @@ static void clear_IO_APIC (void) clear_IO_APIC_pin(apic, pin); } +static void set_ioapic_affinity (unsigned int irq, unsigned long mask) +{ + unsigned long flags; + + /* pick a single cpu for clustered xapics */ + if(clustered_apic_mode == CLUSTERED_APIC_XAPIC){ + int cpu = ffs(mask)-1; + mask = cpu_to_physical_apicid(cpu); + } + + /* + * Only the first 8 bits are valid. + */ + mask = mask << 24; + + spin_lock_irqsave(&ioapic_lock, flags); + __DO_ACTION(1, = mask, ) + spin_unlock_irqrestore(&ioapic_lock, flags); +} + +#ifdef CONFIG_SMP + +#define IRQ_BALANCE_INTERVAL (HZ/50) + +typedef struct { + unsigned int cpu; + unsigned long timestamp; +} ____cacheline_aligned irq_balance_t; + +static irq_balance_t irq_balance[NR_IRQS] __cacheline_aligned; +extern unsigned long irq_affinity [NR_IRQS]; + +#define IRQ_ALLOWED(cpu,allowed_mask) \ + ((1UL << cpu) & (allowed_mask)) + +#define ksoftirqd_is_running(phys_id) (cpu_curr(phys_id) == ksoftirqd_task(phys_id)) +#define __irq_idle_cpu(phys_id) (idle_cpu(phys_id) || ksoftirqd_is_running(phys_id)) +#define irq_idle_cpu(phys_id) (__irq_idle_cpu(phys_id) && \ + (smp_num_siblings <= 1 || __irq_idle_cpu(cpu_sibling_map[phys_id]))) + +static unsigned long move(unsigned int curr_cpu, unsigned long allowed_mask, unsigned long now, int direction) +{ + unsigned int cpu = curr_cpu; + unsigned int phys_id; + + phys_id = cpu_logical_map(cpu); + if (IRQ_ALLOWED(phys_id, allowed_mask) && irq_idle_cpu(phys_id)) + return cpu; + + goto inside; + + do { + if (unlikely(cpu == curr_cpu)) + return cpu; +inside: + if (direction == 1) { + cpu++; + if (cpu >= smp_num_cpus) + cpu = 0; + } else { + cpu--; + if (cpu == -1) + cpu = smp_num_cpus-1; + } + + phys_id = cpu_logical_map(cpu); + } while (!IRQ_ALLOWED(phys_id, allowed_mask) || !irq_idle_cpu(phys_id)); + + return cpu; +} + +#endif /* CONFIG_SMP */ + +static inline void balance_irq(int irq) +{ +#if CONFIG_SMP + irq_balance_t *entry = irq_balance + irq; + unsigned long now = jiffies; + + if (unlikely(time_after(now, entry->timestamp + IRQ_BALANCE_INTERVAL))) { + unsigned long allowed_mask; + unsigned int new_cpu; + int random_number; + + entry->timestamp = now; + + rdtscl(random_number); + random_number &= 1; + + allowed_mask = cpu_online_map & irq_affinity[irq]; + new_cpu = move(entry->cpu, allowed_mask, now, random_number); + if (entry->cpu != new_cpu) { + entry->cpu = new_cpu; + set_ioapic_affinity(irq, clustered_apic_mode == 0 ? + 1UL << new_cpu : cpu_present_to_apicid(entry->cpu) ); + } + } +#endif +} + /* * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to * specific CPU-side IRQs. @@ -703,8 +804,7 @@ void __init setup_IO_APIC_irqs(void) } /* - * Set up the 8259A-master output pin as broadcast to all - * CPUs. + * Set up the 8259A-master output pin: */ void __init setup_ExtINT_IRQ0_pin(unsigned int pin, int vector) { @@ -1256,6 +1356,7 @@ static unsigned int startup_edge_ioapic_ */ static void ack_edge_ioapic_irq(unsigned int irq) { + balance_irq(irq); if ((irq_desc[irq].status & (IRQ_PENDING | IRQ_DISABLED)) == (IRQ_PENDING | IRQ_DISABLED)) mask_IO_APIC_irq(irq); @@ -1295,6 +1396,7 @@ static void end_level_ioapic_irq (unsign unsigned long v; int i; + balance_irq(irq); /* * It appears there is an erratum which affects at least version 0x11 * of I/O APIC (that's the 82093AA and cores integrated into various @@ -1370,26 +1472,6 @@ void send_IPI_self(int vector) #endif /* CONFIG_SMP */ -static void set_ioapic_affinity (unsigned int irq, unsigned long mask) -{ - unsigned long flags; - - /* pick a single cpu for clustered xapics */ - if(clustered_apic_mode == CLUSTERED_APIC_XAPIC){ - int cpu = ffs(mask)-1; - mask = cpu_to_physical_apicid(cpu); - } - - /* - * Only the first 8 bits are valid. - */ - mask = mask << 24; - - spin_lock_irqsave(&ioapic_lock, flags); - __DO_ACTION(1, = mask, ) - spin_unlock_irqrestore(&ioapic_lock, flags); -} - /* * Level and edge triggered IO-APIC interrupts need different handling, * so we use two separate IRQ descriptors. Edge triggered IRQs can be diff -urNp --exclude CVS --exclude BitKeeper x-ref/arch/i386/kernel/irq.c x/arch/i386/kernel/irq.c --- x-ref/arch/i386/kernel/irq.c 2003-09-23 03:15:46.000000000 +0200 +++ x/arch/i386/kernel/irq.c 2003-09-23 03:15:50.000000000 +0200 @@ -1094,7 +1094,7 @@ out: static struct proc_dir_entry * smp_affinity_entry [NR_IRQS]; -static unsigned long irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = ~0UL }; +unsigned long irq_affinity [NR_IRQS] = { [0 ... NR_IRQS-1] = ~0UL }; static int irq_affinity_read_proc (char *page, char **start, off_t off, int count, int *eof, void *data) {