1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
|
From 2bdbdf64f5a227572c323d485f9fcf24f022e971 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@elte.hu>
Date: Fri, 3 Jul 2009 08:29:34 -0500
Subject: [PATCH] timers: prepare for full preemption
commit 6dce6f3f6e182484e48164246f4a268aa42fd19c in tip.
When softirqs can be preempted we need to make sure that cancelling
the timer from the active thread can not deadlock vs. a running timer
callback. Add a waitqueue to resolve that.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
---
include/linux/timer.h | 4 ++-
kernel/timer.c | 97 +++++++++++++++++++++++++++++++++++++++++++------
2 files changed, 88 insertions(+), 13 deletions(-)
diff --git a/include/linux/timer.h b/include/linux/timer.h
index a2d1eb6..af92a91 100644
--- a/include/linux/timer.h
+++ b/include/linux/timer.h
@@ -225,10 +225,12 @@ static inline void timer_stats_timer_clear_start_info(struct timer_list *timer)
extern void add_timer(struct timer_list *timer);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_SOFTIRQS)
+ extern int timer_pending_sync(struct timer_list *timer);
extern int try_to_del_timer_sync(struct timer_list *timer);
extern int del_timer_sync(struct timer_list *timer);
#else
+# define timer_pending_sync(t) timer_pending(t)
# define try_to_del_timer_sync(t) del_timer(t)
# define del_timer_sync(t) del_timer(t)
#endif
diff --git a/kernel/timer.c b/kernel/timer.c
index aeb6a54..e368fe7 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -34,6 +34,7 @@
#include <linux/posix-timers.h>
#include <linux/cpu.h>
#include <linux/syscalls.h>
+#include <linux/kallsyms.h>
#include <linux/delay.h>
#include <linux/tick.h>
#include <linux/kallsyms.h>
@@ -75,6 +76,7 @@ struct tvec_root {
struct tvec_base {
spinlock_t lock;
struct timer_list *running_timer;
+ wait_queue_head_t wait_for_running_timer;
unsigned long timer_jiffies;
unsigned long next_timer;
struct tvec_root tv1;
@@ -323,9 +325,7 @@ EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
static inline void set_running_timer(struct tvec_base *base,
struct timer_list *timer)
{
-#ifdef CONFIG_SMP
base->running_timer = timer;
-#endif
}
static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
@@ -657,6 +657,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
debug_activate(timer, expires);
+ preempt_disable();
cpu = smp_processor_id();
#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
@@ -667,6 +668,8 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
cpu = preferred_cpu;
}
#endif
+ preempt_enable();
+
new_base = per_cpu(tvec_bases, cpu);
if (base != new_base) {
@@ -826,6 +829,18 @@ void add_timer_on(struct timer_list *timer, int cpu)
}
EXPORT_SYMBOL_GPL(add_timer_on);
+/*
+ * Wait for a running timer
+ */
+void wait_for_running_timer(struct timer_list *timer)
+{
+ struct tvec_base *base = timer->base;
+
+ if (base->running_timer == timer)
+ wait_event(base->wait_for_running_timer,
+ base->running_timer != timer);
+}
+
/**
* del_timer - deactive a timer.
* @timer: the timer to be deactivated
@@ -860,7 +875,34 @@ int del_timer(struct timer_list *timer)
}
EXPORT_SYMBOL(del_timer);
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_SOFTIRQS)
+/*
+ * This function checks whether a timer is active and not running on any
+ * CPU. Upon successful (ret >= 0) exit the timer is not queued and the
+ * handler is not running on any CPU.
+ *
+ * It must not be called from interrupt contexts.
+ */
+int timer_pending_sync(struct timer_list *timer)
+{
+ struct tvec_base *base;
+ unsigned long flags;
+ int ret = -1;
+
+ base = lock_timer_base(timer, &flags);
+
+ if (base->running_timer == timer)
+ goto out;
+
+ ret = 0;
+ if (timer_pending(timer))
+ ret = 1;
+out:
+ spin_unlock_irqrestore(&base->lock, flags);
+
+ return ret;
+}
+
/**
* try_to_del_timer_sync - Try to deactivate a timer
* @timer: timer do del
@@ -929,7 +971,7 @@ int del_timer_sync(struct timer_list *timer)
int ret = try_to_del_timer_sync(timer);
if (ret >= 0)
return ret;
- cpu_relax();
+ wait_for_running_timer(timer);
}
}
EXPORT_SYMBOL(del_timer_sync);
@@ -974,6 +1016,20 @@ static inline void __run_timers(struct tvec_base *base)
struct list_head *head = &work_list;
int index = base->timer_jiffies & TVR_MASK;
+ if (softirq_need_resched()) {
+ spin_unlock_irq(&base->lock);
+ wake_up(&base->wait_for_running_timer);
+ cond_resched_softirq_context();
+ cpu_relax();
+ spin_lock_irq(&base->lock);
+ /*
+ * We can simply continue after preemption, nobody
+ * else can touch timer_jiffies so 'index' is still
+ * valid. Any new jiffy will be taken care of in
+ * subsequent loops:
+ */
+ }
+
/*
* Cascade timers:
*/
@@ -1029,18 +1085,17 @@ static inline void __run_timers(struct tvec_base *base)
lock_map_release(&lockdep_map);
if (preempt_count != preempt_count()) {
- printk(KERN_ERR "huh, entered %p "
- "with preempt_count %08x, exited"
- " with %08x?\n",
- fn, preempt_count,
- preempt_count());
- BUG();
+ print_symbol("BUG: unbalanced timer-handler preempt count in %s!\n", (unsigned long) fn);
+ printk("entered with %08x, exited with %08x.\n", preempt_count, preempt_count());
+ preempt_count() = preempt_count;
}
}
+ set_running_timer(base, NULL);
+ cond_resched_softirq_context();
spin_lock_irq(&base->lock);
}
}
- set_running_timer(base, NULL);
+ wake_up(&base->wait_for_running_timer);
spin_unlock_irq(&base->lock);
}
@@ -1206,12 +1261,30 @@ void update_process_times(int user_tick)
}
/*
+ * Time of day handling:
+ */
+static inline void update_times(void)
+{
+ static unsigned long last_tick = INITIAL_JIFFIES;
+ unsigned long ticks, flags;
+
+ write_raw_seqlock_irqsave(&xtime_lock, flags);
+ ticks = jiffies - last_tick;
+ if (ticks) {
+ last_tick += ticks;
+ update_wall_time();
+ }
+ write_raw_sequnlock_irqrestore(&xtime_lock, flags);
+}
+
+/*
* This function runs timers and the timer-tq in bottom half context.
*/
static void run_timer_softirq(struct softirq_action *h)
{
struct tvec_base *base = __get_cpu_var(tvec_bases);
+ update_times();
hrtimer_run_pending();
if (time_after_eq(jiffies, base->timer_jiffies))
@@ -1237,7 +1310,6 @@ void run_local_timers(void)
void do_timer(unsigned long ticks)
{
jiffies_64 += ticks;
- update_wall_time();
calc_global_load();
}
@@ -1552,6 +1624,7 @@ static int __cpuinit init_timers_cpu(int cpu)
}
spin_lock_init(&base->lock);
+ init_waitqueue_head(&base->wait_for_running_timer);
for (j = 0; j < TVN_SIZE; j++) {
INIT_LIST_HEAD(base->tv5.vec + j);
--
1.7.0.4
|
