From: Rusty Russell Now we've moved the bogolock code out to stop_machine.c and generalized it a little, use it in module.c and delete the duplicate code there. --- kernel/module.c | 180 ++++---------------------------------------------------- 1 files changed, 13 insertions(+), 167 deletions(-) diff -puN kernel/module.c~hotplugcpu-use-bogolock-in-modules kernel/module.c --- 25/kernel/module.c~hotplugcpu-use-bogolock-in-modules 2004-02-25 02:32:48.000000000 -0800 +++ 25-akpm/kernel/module.c 2004-02-25 02:32:48.000000000 -0800 @@ -33,7 +33,7 @@ #include #include #include -#include +#include #include #include #include @@ -474,190 +474,36 @@ static inline int try_force(unsigned int } #endif /* CONFIG_MODULE_FORCE_UNLOAD */ -static int try_stop_module_local(struct module *mod, int flags, int *forced) -{ - local_irq_disable(); - - /* If it's not unused, quit unless we are told to block. */ - if ((flags & O_NONBLOCK) && module_refcount(mod) != 0) { - if (!(*forced = try_force(flags))) { - local_irq_enable(); - return -EWOULDBLOCK; - } - } - - /* Mark it as dying. */ - mod->waiter = current; - mod->state = MODULE_STATE_GOING; - local_irq_enable(); - return 0; -} - -#ifdef CONFIG_SMP -/* Thread to stop each CPU in user context. */ -enum stopref_state { - STOPREF_WAIT, - STOPREF_PREPARE, - STOPREF_DISABLE_IRQ, - STOPREF_EXIT, -}; - -static enum stopref_state stopref_state; -static unsigned int stopref_num_threads; -static atomic_t stopref_thread_ack; - -static int stopref(void *cpu) -{ - int irqs_disabled = 0; - int prepared = 0; - - set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu)); - - /* Ack: we are alive */ - mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ - atomic_inc(&stopref_thread_ack); - - /* Simple state machine */ - while (stopref_state != STOPREF_EXIT) { - if (stopref_state == STOPREF_DISABLE_IRQ && !irqs_disabled) { - local_irq_disable(); - irqs_disabled = 1; - /* Ack: irqs disabled. */ - mb(); /* Must read state first. */ - atomic_inc(&stopref_thread_ack); - } else if (stopref_state == STOPREF_PREPARE && !prepared) { - /* Everyone is in place, hold CPU. */ - preempt_disable(); - prepared = 1; - mb(); /* Must read state first. */ - atomic_inc(&stopref_thread_ack); - } - if (irqs_disabled || prepared) - cpu_relax(); - else - yield(); - } - - /* Ack: we are exiting. */ - mb(); /* Must read state first. */ - atomic_inc(&stopref_thread_ack); - - if (irqs_disabled) - local_irq_enable(); - if (prepared) - preempt_enable(); - - return 0; -} - -/* Change the thread state */ -static void stopref_set_state(enum stopref_state state, int sleep) -{ - atomic_set(&stopref_thread_ack, 0); - wmb(); - stopref_state = state; - while (atomic_read(&stopref_thread_ack) != stopref_num_threads) { - if (sleep) - yield(); - else - cpu_relax(); - } -} - struct stopref { struct module *mod; int flags; int *forced; - struct completion started; }; -static int spawn_stopref(void *data) +/* Whole machine is stopped with interrupts off when this runs. */ +static inline int __try_stop_module(void *_sref) { - struct stopref *sref = data; - struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; - unsigned int i, cpu = smp_processor_id(); - int ret = 0; - - complete(&sref->started); - - /* One high-prio thread per cpu. We'll do one (any one). */ - set_cpus_allowed(current, cpumask_of_cpu(cpu)); - sys_sched_setscheduler(current->pid, SCHED_FIFO, ¶m); - - atomic_set(&stopref_thread_ack, 0); - stopref_num_threads = 0; - stopref_state = STOPREF_WAIT; - - for_each_online_cpu(i) { - if (i == cpu) - continue; - ret = kernel_thread(stopref, (void *)(long)i, CLONE_KERNEL); - if (ret < 0) - break; - stopref_num_threads++; - } - - /* Wait for them all to come to life. */ - while (atomic_read(&stopref_thread_ack) != stopref_num_threads) - yield(); + struct stopref *sref = _sref; - /* If some failed, kill them all. */ - if (ret < 0) { - stopref_set_state(STOPREF_EXIT, 1); - goto out; + /* If it's not unused, quit unless we are told to block. */ + if ((sref->flags & O_NONBLOCK) && module_refcount(sref->mod) != 0) { + if (!(*sref->forced = try_force(sref->flags))) + return -EWOULDBLOCK; } - /* Don't schedule us away at this point, please. */ - preempt_disable(); - - /* Now they are all started, make them hold the CPUs, ready. */ - stopref_set_state(STOPREF_PREPARE, 0); - - /* Make them disable irqs. */ - stopref_set_state(STOPREF_DISABLE_IRQ, 0); - - /* Atomically disable module if possible */ - ret = try_stop_module_local(sref->mod, sref->flags, sref->forced); - - stopref_set_state(STOPREF_EXIT, 0); - preempt_enable(); - -out: - /* Wait for kthread_stop */ - while (!kthread_should_stop()) { - __set_current_state(TASK_INTERRUPTIBLE); - schedule(); - } - return ret; + /* Mark it as dying. */ + sref->mod->waiter = current; + sref->mod->state = MODULE_STATE_GOING; + return 0; } static int try_stop_module(struct module *mod, int flags, int *forced) { - struct task_struct *p; struct stopref sref = { mod, flags, forced }; - int ret; - - init_completion(&sref.started); - /* No CPUs can come up or down during this. */ - lock_cpu_hotplug(); - p = kthread_run(spawn_stopref, &sref, "krmmod"); - if (IS_ERR(p)) - ret = PTR_ERR(p); - else { - wait_for_completion(&sref.started); - ret = kthread_stop(p); - } - unlock_cpu_hotplug(); - return ret; + return stop_machine_run(__try_stop_module, &sref, NR_CPUS); } -#else /* ...!SMP */ -static inline int try_stop_module(struct module *mod, int flags, int *forced) -{ - return try_stop_module_local(mod, flags, forced); -} -#endif unsigned int module_refcount(struct module *mod) { _