1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
From a6a25d4559c05d4549137edcff77f4ffdb8ddb7f Mon Sep 17 00:00:00 2001
From: Peter Zijlstra <a.p.zijlstra@chello.nl>
Date: Thu, 11 Aug 2011 15:03:35 +0200
Subject: [PATCH 147/270] sched: Optimize migrate_disable
Change from task_rq_lock() to raw_spin_lock(&rq->lock) to avoid a few
atomic ops. See comment on why it should be safe.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/n/tip-cbz6hkl5r5mvwtx5s3tor2y6@git.kernel.org
---
kernel/sched/core.c | 24 ++++++++++++++++++++----
1 file changed, 20 insertions(+), 4 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e75dae8..ba738aa 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5208,7 +5208,19 @@ void migrate_disable(void)
preempt_enable();
return;
}
- rq = task_rq_lock(p, &flags);
+
+ /*
+ * Since this is always current we can get away with only locking
+ * rq->lock, the ->cpus_allowed value can normally only be changed
+ * while holding both p->pi_lock and rq->lock, but seeing that this
+ * it current, we cannot actually be waking up, so all code that
+ * relies on serialization against p->pi_lock is out of scope.
+ *
+ * Taking rq->lock serializes us against things like
+ * set_cpus_allowed_ptr() that can still happen concurrently.
+ */
+ rq = this_rq();
+ raw_spin_lock_irqsave(&rq->lock, flags);
p->migrate_disable = 1;
mask = tsk_cpus_allowed(p);
@@ -5219,7 +5231,7 @@ void migrate_disable(void)
p->sched_class->set_cpus_allowed(p, mask);
p->rt.nr_cpus_allowed = cpumask_weight(mask);
}
- task_rq_unlock(rq, p, &flags);
+ raw_spin_unlock_irqrestore(&rq->lock, flags);
preempt_enable();
}
EXPORT_SYMBOL(migrate_disable);
@@ -5247,7 +5259,11 @@ void migrate_enable(void)
return;
}
- rq = task_rq_lock(p, &flags);
+ /*
+ * See comment in migrate_disable().
+ */
+ rq = this_rq();
+ raw_spin_lock_irqsave(&rq->lock, flags);
p->migrate_disable = 0;
mask = tsk_cpus_allowed(p);
@@ -5259,7 +5275,7 @@ void migrate_enable(void)
p->rt.nr_cpus_allowed = cpumask_weight(mask);
}
- task_rq_unlock(rq, p, &flags);
+ raw_spin_unlock_irqrestore(&rq->lock, flags);
unpin_current_cpu();
preempt_enable();
}
--
1.7.10.4
|
