From: "David S. Miller" Ok, here is a patch that gets sparc64 compiling and booting again. But I have some reservations about these changes. Many things got lost in the translation and the code being output should look exactly like what we got before cpumask_t but that's not happening. 1) All of my modifications of mm->cpu_vm_mask on sparc64 we done under the mm_struct lock, therefore locked bitops are superfluous. I have no way to continue this optimization in the new code. We really need __cpu_set() et al. operations that use the non-atomic bitops. This should be easy to do. 2) for NR_CPUS < 64, cpumask_t should simply typedef to an unsigned long, that is the only way the code will be as efficient as it is now for these cases. The biggest area of concern is when cpumask_t's are passed on the stack, probably these should be changed to cpumask_t *'s but that's a regression from the current "unsigned long" scheme. This pointer idea may also fail in cases where the caller really wants his copy of the mask to remain unmodified. For example, look at how the code now looks in arch/sparc64/kernel/smp.c:cheetah_xcall_deliver() Here, I have two masks to maintain, with structs this code is output in an absolutely horrible manner. With unsigned long masks it's actually reasonably looking. Another concern is cpus_empty(), for NR_CPUS % BITS_PER_LONG the code output is just rediculious. There are some other considerations. People put these cpumask_t things on the stack, not a real problem now but something to keep in mind as we up the limit. Next, some ports save the IRQ affinity settings from /proc in the irqaction ->mask field. This no longer works, and I'd suggest dealing with this by putting a cpumask_t into struct irqaction or even better a irqaction_arch_t object of some sort. arch/sparc64/kernel/irq.c | 6 +++- arch/sparc64/kernel/smp.c | 54 ++++++++++++++++++------------------ arch/sparc64/kernel/sparc64_ksyms.c | 1 arch/sparc64/kernel/us2e_cpufreq.c | 5 ++- arch/sparc64/kernel/us3_cpufreq.c | 5 ++- include/asm-sparc64/bitops.h | 9 ++++++ include/asm-sparc64/mmu_context.h | 14 ++++----- include/asm-sparc64/smp.h | 6 ---- kernel/module.c | 2 - 9 files changed, 56 insertions(+), 46 deletions(-) diff -puN arch/sparc64/kernel/irq.c~sparc64-cpumask_t-fix arch/sparc64/kernel/irq.c --- 25/arch/sparc64/kernel/irq.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/arch/sparc64/kernel/irq.c 2003-07-10 18:59:20.000000000 -0700 @@ -110,6 +110,10 @@ static void register_irq_proc (unsigned action->flags |= __irq_ino(irq) << 48; #define get_ino_in_irqaction(action) (action->flags >> 48) +#if NR_CPUS > 64 +#error irqaction embedded smp affinity does not work with > 64 cpus, FIXME +#endif + #define put_smpaff_in_irqaction(action, smpaff) (action)->mask = (smpaff) #define get_smpaff_in_irqaction(action) ((action)->mask) @@ -658,7 +662,7 @@ static inline void redirect_intr(int cpu * Just Do It. */ struct irqaction *ap = bp->irq_info; - cpumask_t cpu_mask = get_smpaff_in_irqaction(ap); + cpumask_t cpu_mask = { .mask[0] = get_smpaff_in_irqaction(ap) }; unsigned int buddy, ticks; cpus_and(cpu_mask, cpu_mask, cpu_online_map); diff -puN arch/sparc64/kernel/smp.c~sparc64-cpumask_t-fix arch/sparc64/kernel/smp.c --- 25/arch/sparc64/kernel/smp.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/arch/sparc64/kernel/smp.c 2003-07-10 18:59:20.000000000 -0700 @@ -46,8 +46,7 @@ cpuinfo_sparc cpu_data[NR_CPUS]; /* Please don't make this stuff initdata!!! --DaveM */ static unsigned char boot_cpu_id; -atomic_t sparc64_num_cpus_online = ATOMIC_INIT(0); -unsigned long cpu_online_map = CPU_MASK_NONE; +cpumask_t cpu_online_map = CPU_MASK_NONE; atomic_t sparc64_num_cpus_possible = ATOMIC_INIT(0); cpumask_t phys_cpu_present_map = CPU_MASK_NONE; static cpumask_t smp_commenced_mask; @@ -155,7 +154,6 @@ void __init smp_callin(void) membar("#LoadLoad"); cpu_set(cpuid, cpu_online_map); - atomic_inc(&sparc64_num_cpus_online); } void cpu_panic(void) @@ -420,17 +418,17 @@ again: } } -static __inline__ void spitfire_xcall_deliver(u64 data0, u64 data1, u64 data2, unsigned long mask) +static __inline__ void spitfire_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask) { u64 pstate; int i; __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); for (i = 0; i < NR_CPUS; i++) { - if (mask & (1UL << i)) { + if (cpu_isset(i, mask)) { spitfire_xcall_helper(data0, data1, data2, pstate, i); - mask &= ~(1UL << i); - if (!mask) + cpu_clear(i, mask); + if (cpus_empty(mask)) break; } } @@ -443,12 +441,12 @@ static __inline__ void spitfire_xcall_de #if NR_CPUS > 32 #error Fixup cheetah_xcall_deliver Dave... #endif -static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, unsigned long mask) +static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask) { u64 pstate; int nack_busy_id; - if (!mask) + if (cpus_empty(mask)) return; __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); @@ -469,11 +467,11 @@ retry: nack_busy_id = 0; { - unsigned long work_mask = mask; + cpumask_t work_mask = mask; int i; for (i = 0; i < NR_CPUS; i++) { - if (work_mask & (1UL << i)) { + if (cpu_isset(i, work_mask)) { u64 target = (i << 14) | 0x70; target |= (nack_busy_id++ << 24); @@ -482,8 +480,8 @@ retry: "membar #Sync\n\t" : /* no outputs */ : "r" (target), "i" (ASI_INTR_W)); - work_mask &= ~(1UL << i); - if (!work_mask) + cpu_clear(i, work_mask); + if (cpus_empty(work_mask)) break; } } @@ -518,7 +516,7 @@ retry: printk("CPU[%d]: mondo stuckage result[%016lx]\n", smp_processor_id(), dispatch_stat); } else { - unsigned long work_mask = mask; + cpumask_t work_mask = mask; int i, this_busy_nack = 0; /* Delay some random time with interrupts enabled @@ -530,13 +528,13 @@ retry: * NACK us. */ for (i = 0; i < NR_CPUS; i++) { - if (work_mask & (1UL << i)) { + if (cpu_isset(i, work_mask)) { if ((dispatch_stat & (0x2 << this_busy_nack)) == 0) - mask &= ~(1UL << i); + cpu_clear(i, mask); this_busy_nack += 2; - work_mask &= ~(1UL << i); - if (!work_mask) + cpu_clear(i, work_mask); + if (cpus_empty(work_mask)) break; } } @@ -567,9 +565,10 @@ extern unsigned long xcall_sync_tick; static void smp_start_sync_tick_client(int cpu) { + cpumask_t mask = cpumask_of_cpu(cpu); + smp_cross_call_masked(&xcall_sync_tick, - 0, 0, 0, - (1UL << cpu)); + 0, 0, 0, mask); } /* Send cross call to all processors except self. */ @@ -751,7 +750,7 @@ void flush_dcache_page_all(struct mm_str void smp_receive_signal(int cpu) { - unsigned long mask = cpumask_of_cpu(cpu); + cpumask_t mask = cpumask_of_cpu(cpu); if (cpu_online(cpu)) { u64 data0 = (((u64)&xcall_receive_signal) & 0xffffffff); @@ -855,7 +854,7 @@ void smp_flush_tlb_mm(struct mm_struct * if (atomic_read(&mm->mm_users) == 1) { /* See smp_flush_tlb_page for info about this. */ - mm->cpu_vm_mask = (1UL << cpu); + mm->cpu_vm_mask = cpumask_of_cpu(cpu); goto local_flush_and_out; } @@ -878,7 +877,7 @@ void smp_flush_tlb_range(struct mm_struc end = PAGE_ALIGN(end); if (mm == current->active_mm && atomic_read(&mm->mm_users) == 1) { - mm->cpu_vm_mask = (1UL << cpu); + mm->cpu_vm_mask = cpumask_of_cpu(cpu); goto local_flush_and_out; } @@ -922,14 +921,16 @@ void smp_flush_tlb_page(struct mm_struct * is almost certain that all TLB entries for this * context will be replaced by the time that happens. */ - mm->cpu_vm_mask = (1UL << cpu); + mm->cpu_vm_mask = cpumask_of_cpu(cpu); goto local_flush_and_out; } else { + cpumask_t this_cpu_mask = cpumask_of_cpu(cpu); + /* By virtue of running under the mm->page_table_lock, * and mmu_context.h:switch_mm doing the same, the * following operation is safe. */ - if (mm->cpu_vm_mask == (1UL << cpu)) + if (cpus_equal(mm->cpu_vm_mask, this_cpu_mask)) goto local_flush_and_out; } @@ -940,7 +941,7 @@ void smp_flush_tlb_page(struct mm_struct smp_cross_call_masked(&xcall_flush_tlb_page, ctx, page, 0, mm->cpu_vm_mask); - if (!(mm->cpu_vm_mask & (1UL << cpu))) + if (!cpu_isset(cpu, mm->cpu_vm_mask)) return; local_flush_and_out: @@ -1123,7 +1124,6 @@ void __init smp_tick_init(void) prom_halt(); } - atomic_inc(&sparc64_num_cpus_online); cpu_set(boot_cpu_id, cpu_online_map); prom_cpu_nodes[boot_cpu_id] = linux_cpus[0].prom_node; prof_counter(boot_cpu_id) = prof_multiplier(boot_cpu_id) = 1; diff -puN arch/sparc64/kernel/sparc64_ksyms.c~sparc64-cpumask_t-fix arch/sparc64/kernel/sparc64_ksyms.c --- 25/arch/sparc64/kernel/sparc64_ksyms.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/arch/sparc64/kernel/sparc64_ksyms.c 2003-07-10 18:59:20.000000000 -0700 @@ -147,7 +147,6 @@ EXPORT_SYMBOL(cpu_data); /* CPU online map and active count. */ EXPORT_SYMBOL(cpu_online_map); -EXPORT_SYMBOL(sparc64_num_cpus_online); EXPORT_SYMBOL(phys_cpu_present_map); EXPORT_SYMBOL(sparc64_num_cpus_possible); diff -puN arch/sparc64/kernel/us2e_cpufreq.c~sparc64-cpumask_t-fix arch/sparc64/kernel/us2e_cpufreq.c --- 25/arch/sparc64/kernel/us2e_cpufreq.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/arch/sparc64/kernel/us2e_cpufreq.c 2003-07-10 18:59:20.000000000 -0700 @@ -232,15 +232,16 @@ static unsigned long estar_to_divisor(un static void us2e_set_cpu_divider_index(unsigned int cpu, unsigned int index) { - unsigned long new_bits, new_freq, cpus_allowed; + unsigned long new_bits, new_freq; unsigned long clock_tick, divisor, old_divisor, estar; + cpumask_t cpus_allowed; struct cpufreq_freqs freqs; if (!cpu_online(cpu)) return; cpus_allowed = current->cpus_allowed; - set_cpus_allowed(current, (1UL << cpu)); + set_cpus_allowed(current, cpumask_of_cpu(cpu)); new_freq = clock_tick = sparc64_get_clock_tick(cpu); new_bits = index_to_estar_mode(index); diff -puN arch/sparc64/kernel/us3_cpufreq.c~sparc64-cpumask_t-fix arch/sparc64/kernel/us3_cpufreq.c --- 25/arch/sparc64/kernel/us3_cpufreq.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/arch/sparc64/kernel/us3_cpufreq.c 2003-07-10 18:59:20.000000000 -0700 @@ -78,14 +78,15 @@ static unsigned long get_current_freq(un static void us3_set_cpu_divider_index(unsigned int cpu, unsigned int index) { - unsigned long new_bits, new_freq, reg, cpus_allowed; + unsigned long new_bits, new_freq, reg; + cpumask_t cpus_allowed; struct cpufreq_freqs freqs; if (!cpu_online(cpu)) return; cpus_allowed = current->cpus_allowed; - set_cpus_allowed(current, (1UL << cpu)); + set_cpus_allowed(current, cpumask_of_cpu(cpu)); new_freq = sparc64_get_clock_tick(cpu); switch (index) { diff -puN include/asm-sparc64/bitops.h~sparc64-cpumask_t-fix include/asm-sparc64/bitops.h --- 25/include/asm-sparc64/bitops.h~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/include/asm-sparc64/bitops.h 2003-07-10 18:59:20.000000000 -0700 @@ -156,6 +156,14 @@ static __inline__ int ffs(int x) #ifdef ULTRA_HAS_POPULATION_COUNT +static __inline__ unsigned int hweight64(unsigned long w) +{ + unsigned int res; + + __asm__ ("popc %1,%0" : "=r" (res) : "r" (w)); + return res; +} + static __inline__ unsigned int hweight32(unsigned int w) { unsigned int res; @@ -182,6 +190,7 @@ static __inline__ unsigned int hweight8( #else +#define hweight64(x) generic_hweight64(x) #define hweight32(x) generic_hweight32(x) #define hweight16(x) generic_hweight16(x) #define hweight8(x) generic_hweight8(x) diff -puN include/asm-sparc64/mmu_context.h~sparc64-cpumask_t-fix include/asm-sparc64/mmu_context.h --- 25/include/asm-sparc64/mmu_context.h~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/include/asm-sparc64/mmu_context.h 2003-07-10 18:59:20.000000000 -0700 @@ -125,7 +125,7 @@ static inline void switch_mm(struct mm_s } { - unsigned long vm_mask = (1UL << smp_processor_id()); + int cpu = smp_processor_id(); /* Even if (mm == old_mm) we _must_ check * the cpu_vm_mask. If we do not we could @@ -133,8 +133,8 @@ static inline void switch_mm(struct mm_s * smp_flush_tlb_{page,range,mm} on sparc64 * and lazy tlb switches work. -DaveM */ - if (!ctx_valid || !(mm->cpu_vm_mask & vm_mask)) { - mm->cpu_vm_mask |= vm_mask; + if (!ctx_valid || !cpu_isset(cpu, mm->cpu_vm_mask)) { + cpu_set(cpu, mm->cpu_vm_mask); __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT); } } @@ -148,14 +148,14 @@ extern void __flush_tlb_mm(unsigned long /* Activate a new MM instance for the current task. */ static inline void activate_mm(struct mm_struct *active_mm, struct mm_struct *mm) { - unsigned long vm_mask; + int cpu; spin_lock(&mm->page_table_lock); if (!CTX_VALID(mm->context)) get_new_mmu_context(mm); - vm_mask = (1UL << smp_processor_id()); - if (!(mm->cpu_vm_mask & vm_mask)) - mm->cpu_vm_mask |= vm_mask; + cpu = smp_processor_id(); + if (!cpu_isset(cpu, mm->cpu_vm_mask)) + cpu_set(cpu, mm->cpu_vm_mask); spin_unlock(&mm->page_table_lock); load_secondary_context(mm); diff -puN include/asm-sparc64/smp.h~sparc64-cpumask_t-fix include/asm-sparc64/smp.h --- 25/include/asm-sparc64/smp.h~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/include/asm-sparc64/smp.h 2003-07-10 18:59:20.000000000 -0700 @@ -8,13 +8,13 @@ #include #include -#include #include #include #include #ifndef __ASSEMBLY__ +#include #include /* PROM provided per-processor information we need @@ -72,12 +72,8 @@ extern unsigned char boot_cpu_id; extern cpumask_t phys_cpu_present_map; #define cpu_possible(cpu) cpu_isset(cpu, phys_cpu_present_map) -extern unsigned long cpu_online_map; #define cpu_online(cpu) cpu_isset(cpu, cpu_online_map) -extern atomic_t sparc64_num_cpus_online; -#define num_online_cpus() (atomic_read(&sparc64_num_cpus_online)) - extern atomic_t sparc64_num_cpus_possible; #define num_possible_cpus() (atomic_read(&sparc64_num_cpus_possible)) diff -puN kernel/module.c~sparc64-cpumask_t-fix kernel/module.c --- 25/kernel/module.c~sparc64-cpumask_t-fix 2003-07-10 18:59:20.000000000 -0700 +++ 25-akpm/kernel/module.c 2003-07-10 18:59:20.000000000 -0700 @@ -471,7 +471,7 @@ static int stopref(void *cpu) struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; setscheduler(current->pid, SCHED_FIFO, ¶m); #endif - set_cpus_allowed(current, cpumask_of_cpu((long)cpu)); + set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu)); /* Ack: we are alive */ atomic_inc(&stopref_thread_ack); _