diff options
Diffstat (limited to 'arch/ia64/sn/kernel/sn2/sn2_smp.c')
-rw-r--r-- | arch/ia64/sn/kernel/sn2/sn2_smp.c | 333 |
1 files changed, 18 insertions, 315 deletions
diff --git a/arch/ia64/sn/kernel/sn2/sn2_smp.c b/arch/ia64/sn/kernel/sn2/sn2_smp.c index 1850229970c143..c7116a35e5b141 100644 --- a/arch/ia64/sn/kernel/sn2/sn2_smp.c +++ b/arch/ia64/sn/kernel/sn2/sn2_smp.c @@ -1,7 +1,7 @@ /* * SN2 Platform specific SMP Support * - * Copyright (C) 2000-2002 Silicon Graphics, Inc. All rights reserved. + * Copyright (C) 2000-2003 Silicon Graphics, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of version 2 of the GNU General Public License @@ -79,284 +79,6 @@ wait_piowc(void) return ws; } -#ifdef PTCG_WAR -/* - * The following structure is used to pass params thru smp_call_function - * to other cpus for flushing TLB ranges. - */ -typedef struct { - unsigned long start; - unsigned long end; - unsigned long nbits; - unsigned int rid; - atomic_t unfinished_count; - char fill[96]; -} ptc_params_t; - -#define NUMPTC 512 - -static ptc_params_t ptcParamArray[NUMPTC] __attribute__((__aligned__(128))); - -/* use separate cache lines on ptcParamsNextByCpu to avoid false sharing */ -static ptc_params_t *ptcParamsNextByCpu[NR_CPUS*16] __attribute__((__aligned__(128))); -static volatile ptc_params_t *ptcParamsEmpty __cacheline_aligned; - -/*REFERENCED*/ -static spinlock_t ptcParamsLock __cacheline_aligned = SPIN_LOCK_UNLOCKED; - -static int ptcInit = 0; -#ifdef PTCDEBUG -static int ptcParamsAllBusy = 0; /* debugging/statistics */ -static int ptcCountBacklog = 0; -static int ptcBacklog[NUMPTC+1]; -static char ptcParamsCounts[NR_CPUS][NUMPTC] __attribute__((__aligned__(128))); -static char ptcParamsResults[NR_CPUS][NUMPTC] __attribute__((__aligned__(128))); -#endif - -/* - * Make smp_send_flush_tlbsmp_send_flush_tlb() a weak reference, - * so that we get a clean compile with the ia64 patch without the - * actual SN1 specific code in arch/ia64/kernel/smp.c. - */ -extern void smp_send_flush_tlb (void) __attribute((weak)); - - -/** - * sn1_ptc_l_range - purge local translation cache - * @start: start of virtual address range - * @end: end of virtual address range - * @nbits: specifies number of bytes to purge per instruction (num = 1<<(nbits & 0xfc)) - * - * Purges the range specified from the local processor's translation cache - * (as opposed to the translation registers). Note that more than the specified - * range *may* be cleared from the cache by some processors. - * - * This is probably not good enough, but I don't want to try to make it better - * until I get some statistics on a running system. At a minimum, we should only - * send IPIs to 1 processor in each TLB domain & have it issue a ptc.g on it's - * own FSB. Also, we only have to serialize per FSB, not globally. - * - * More likely, we will have to do some work to reduce the frequency of calls to - * this routine. - */ -static inline void -sn1_ptc_l_range(unsigned long start, unsigned long end, unsigned long nbits) -{ - do { - __asm__ __volatile__ ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory"); - start += (1UL << nbits); - } while (start < end); - ia64_srlz_d(); -} - -/** - * sn1_received_flush_tlb - cpu tlb flush routine - * - * Flushes the TLB of a given processor. - */ -void -sn1_received_flush_tlb(void) -{ - unsigned long start, end, nbits; - unsigned int rid, saved_rid; - int cpu = smp_processor_id(); - int result; - ptc_params_t *ptcParams; - - ptcParams = ptcParamsNextByCpu[cpu*16]; - if (ptcParams == ptcParamsEmpty) - return; - - do { - start = ptcParams->start; - saved_rid = (unsigned int) ia64_get_rr(start); - end = ptcParams->end; - nbits = ptcParams->nbits; - rid = ptcParams->rid; - - if (saved_rid != rid) { - ia64_set_rr(start, (unsigned long)rid); - ia64_srlz_d(); - } - - sn1_ptc_l_range(start, end, nbits); - - if (saved_rid != rid) - ia64_set_rr(start, (unsigned long)saved_rid); - - ia64_srlz_i(); - - result = atomic_dec(&ptcParams->unfinished_count); -#ifdef PTCDEBUG - { - int i = ptcParams-&ptcParamArray[0]; - ptcParamsResults[cpu][i] = (char) result; - ptcParamsCounts[cpu][i]++; - } -#endif /* PTCDEBUG */ - - if (++ptcParams == &ptcParamArray[NUMPTC]) - ptcParams = &ptcParamArray[0]; - - } while (ptcParams != ptcParamsEmpty); - - ptcParamsNextByCpu[cpu*16] = ptcParams; -} - -/** - * sn1_global_tlb_purge - flush a translation cache range on all processors - * @start: start of virtual address range to flush - * @end: end of virtual address range - * @nbits: specifies number of bytes to purge per instruction (num = 1<<(nbits & 0xfc)) - * - * Flushes the translation cache of all processors from @start to @end. - */ -void -sn1_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits) -{ - ptc_params_t *params; - ptc_params_t *next; - unsigned long irqflags; -#ifdef PTCDEBUG - ptc_params_t *nextnext; - int backlog = 0; -#endif - - if (smp_num_cpus == 1) { - sn1_ptc_l_range(start, end, nbits); - return; - } - - if (in_interrupt()) { - /* - * If at interrupt level and cannot get spinlock, - * then do something useful by flushing own tlbflush queue - * so as to avoid a possible deadlock. - */ - while (!spin_trylock(&ptcParamsLock)) { - local_irq_save(irqflags); - sn1_received_flush_tlb(); - local_irq_restore(irqflags); - udelay(10); /* take it easier on the bus */ - } - } else { - spin_lock(&ptcParamsLock); - } - - if (!ptcInit) { - int cpu; - ptcInit = 1; - memset(ptcParamArray, 0, sizeof(ptcParamArray)); - ptcParamsEmpty = &ptcParamArray[0]; - for (cpu=0; cpu<NR_CPUS; cpu++) - ptcParamsNextByCpu[cpu*16] = &ptcParamArray[0]; - -#ifdef PTCDEBUG - memset(ptcBacklog, 0, sizeof(ptcBacklog)); - memset(ptcParamsCounts, 0, sizeof(ptcParamsCounts)); - memset(ptcParamsResults, 0, sizeof(ptcParamsResults)); -#endif /* PTCDEBUG */ - } - - params = (ptc_params_t *) ptcParamsEmpty; - next = (ptc_params_t *) ptcParamsEmpty + 1; - if (next == &ptcParamArray[NUMPTC]) - next = &ptcParamArray[0]; - -#ifdef PTCDEBUG - nextnext = next + 1; - if (nextnext == &ptcParamArray[NUMPTC]) - nextnext = &ptcParamArray[0]; - - if (ptcCountBacklog) { - /* quick count of backlog */ - ptc_params_t *ptr; - - /* check the current pointer to the beginning */ - ptr = params; - while(--ptr >= &ptcParamArray[0]) { - if (atomic_read(&ptr->unfinished_count) == 0) - break; - ++backlog; - } - - if (backlog) { - /* check the end of the array */ - ptr = &ptcParamArray[NUMPTC]; - while (--ptr > params) { - if (atomic_read(&ptr->unfinished_count) == 0) - break; - ++backlog; - } - } - ptcBacklog[backlog]++; - } -#endif /* PTCDEBUG */ - - /* wait for the next entry to clear...should be rare */ - if (atomic_read(&next->unfinished_count) > 0) { -#ifdef PTCDEBUG - ptcParamsAllBusy++; - - if (atomic_read(&nextnext->unfinished_count) == 0) { - if (atomic_read(&next->unfinished_count) > 0) { - panic("\nnonzero next zero nextnext %lx %lx\n", - (long)next, (long)nextnext); - } - } -#endif - - /* it could be this cpu that is behind */ - local_irq_save(irqflags); - sn1_received_flush_tlb(); - local_irq_restore(irqflags); - - /* now we know it's not this cpu, so just wait */ - while (atomic_read(&next->unfinished_count) > 0) { - barrier(); - } - } - - params->start = start; - params->end = end; - params->nbits = nbits; - params->rid = (unsigned int) ia64_get_rr(start); - atomic_set(¶ms->unfinished_count, smp_num_cpus); - - /* The atomic_set above can hit memory *after* the update - * to ptcParamsEmpty below, which opens a timing window - * that other cpus can squeeze into! - */ - mb(); - - /* everything is ready to process: - * -- global lock is held - * -- new entry + 1 is free - * -- new entry is set up - * so now: - * -- update the global next pointer - * -- unlock the global lock - * -- send IPI to notify other cpus - * -- process the data ourselves - */ - ptcParamsEmpty = next; - spin_unlock(&ptcParamsLock); - smp_send_flush_tlb(); - - local_irq_save(irqflags); - sn1_received_flush_tlb(); - local_irq_restore(irqflags); - - /* - * Since IPIs are polled event (for now), we need to wait til the - * TLB flush has started. - * wait for the flush to complete - */ - while (atomic_read(¶ms->unfinished_count) > 0) - barrier(); -} - -#endif /* PTCG_WAR */ /** @@ -372,18 +94,10 @@ sn1_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit void sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits) { - int cnode, mycnode, nasid; + int cnode, mycnode, nasid, flushed=0; volatile unsigned long *ptc0, *ptc1; unsigned long flags=0, data0, data1; - /* - * Special case 1 cpu & 1 node. Use local purges. - */ -#ifdef PTCG_WAR - sn1_global_tlb_purge(start, end, nbits); - return; -#endif /* PTCG_WAR */ - data0 = (1UL<<SH_PTC_0_A_SHFT) | (nbits<<SH_PTC_0_PS_SHFT) | ((ia64_get_rr(start)>>8)<<SH_PTC_0_RID_SHFT) | @@ -392,18 +106,9 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit ptc0 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_0); ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1); - mycnode = local_nodeid; - - /* - * For now, we don't want to spin uninterruptibly waiting - * for the lock. Makes hangs hard to debug. - */ - local_irq_save(flags); - while (!spin_trylock(&sn2_global_ptc_lock)) { - local_irq_restore(flags); - udelay(1); - local_irq_save(flags); - } + mycnode = numa_node_id(); + + spin_lock_irqsave(&sn2_global_ptc_lock, flags); do { data1 = start | (1UL<<SH_PTC_1_START_SHFT); @@ -417,11 +122,13 @@ sn2_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbit ptc0 = CHANGE_NASID(nasid, ptc0); ptc1 = CHANGE_NASID(nasid, ptc1); pio_atomic_phys_write_mmrs(ptc0, data0, ptc1, data1); + flushed = 1; } } - if (wait_piowc() & SH_PIO_WRITE_STATUS_0_WRITE_DEADLOCK_MASK) + if (flushed && (wait_piowc() & SH_PIO_WRITE_STATUS_0_WRITE_DEADLOCK_MASK)) { sn2_ptc_deadlock_recovery(data0, data1); + } start += (1UL << nbits); @@ -451,7 +158,7 @@ sn2_ptc_deadlock_recovery(unsigned long data0, unsigned long data1) ptc1 = (long*)GLOBAL_MMR_PHYS_ADDR(0, SH_PTC_1); piows = (long*)pda->pio_write_status_addr; - mycnode = local_nodeid; + mycnode = numa_node_id(); for (cnode = 0; cnode < numnodes; cnode++) { if (is_headless_node(cnode) || cnode == mycnode) @@ -482,16 +189,10 @@ sn2_ptc_deadlock_recovery(unsigned long data0, unsigned long data1) void sn_send_IPI_phys(long physid, int vector, int delivery_mode) { - long nasid, slice; - long val; + long nasid, slice, val; + unsigned long flags=0; volatile long *p; -#ifdef BUS_INT_WAR - if (vector != ap_wakeup_vector && delivery_mode == IA64_IPI_DM_INT) { - return; - } -#endif - nasid = cpu_physical_id_to_nasid(physid); slice = cpu_physical_id_to_slice(physid); @@ -503,12 +204,15 @@ sn_send_IPI_phys(long physid, int vector, int delivery_mode) (0x000feeUL<<SH_IPI_INT_BASE_SHFT); mb(); + if (enable_shub_wars_1_1() ) { + spin_lock_irqsave(&sn2_global_ptc_lock, flags); + } pio_phys_write_mmr(p, val); + if (enable_shub_wars_1_1() ) { + wait_piowc(); + spin_unlock_irqrestore(&sn2_global_ptc_lock, flags); + } -#ifndef CONFIG_SHUB_1_0_SPECIFIC - /* doesn't work on shub 1.0 */ - wait_piowc(); -#endif } /** @@ -536,4 +240,3 @@ sn2_send_IPI(int cpuid, int vector, int delivery_mode, int redirect) sn_send_IPI_phys(physid, vector, delivery_mode); } - |