1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
|
From f89676f49837d6c07263c84bfa6574d2f85a3d1a Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@elte.hu>
Date: Fri, 3 Jul 2009 08:29:51 -0500
Subject: [PATCH] mm: convert swap to percpu locked
commit f8382688911469d60a31cb2b1ebe378d46dc790e in tip.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
---
mm/swap.c | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++++-------
1 files changed, 94 insertions(+), 13 deletions(-)
diff --git a/mm/swap.c b/mm/swap.c
index 7cd60bf..3f917bb 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -31,15 +31,92 @@
#include <linux/backing-dev.h>
#include <linux/memcontrol.h>
#include <linux/gfp.h>
+#include <linux/interrupt.h>
#include "internal.h"
/* How many pages do we try to swap or page in/out together? */
int page_cluster;
+#ifdef CONFIG_PREEMPT_RT
+/*
+ * On PREEMPT_RT we don't want to disable preemption for cpu variables.
+ * We grab a cpu and then use that cpu to lock the variables accordingly.
+ *
+ * (On !PREEMPT_RT this turns into normal preempt-off sections, as before.)
+ */
+static DEFINE_PER_CPU_LOCKED(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
+static DEFINE_PER_CPU_LOCKED(struct pagevec, lru_rotate_pvecs);
+
+#define swap_get_cpu_var_irq_save(var, flags, cpu) \
+ ({ \
+ (void)flags; \
+ &get_cpu_var_locked(var, &cpu); \
+ })
+
+#define swap_put_cpu_var_irq_restore(var, flags, cpu) \
+ put_cpu_var_locked(var, cpu)
+
+#define swap_get_cpu_var(var, cpu) \
+ &get_cpu_var_locked(var, &cpu)
+
+#define swap_put_cpu_var(var, cpu) \
+ put_cpu_var_locked(var, cpu)
+
+#define swap_per_cpu_lock(var, cpu) \
+ ({ \
+ spin_lock(&__get_cpu_lock(var, cpu)); \
+ &__get_cpu_var_locked(var, cpu); \
+ })
+
+#define swap_per_cpu_unlock(var, cpu) \
+ spin_unlock(&__get_cpu_lock(var, cpu));
+
+#define swap_get_cpu() raw_smp_processor_id()
+
+#define swap_put_cpu() do { } while (0)
+
+#define swap_irq_save(flags) do { (void)flags; } while (0)
+
+#define swap_irq_restore(flags) do { (void)flags; } while (0)
+
+#else
+
static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
+#define swap_get_cpu_var_irq_save(var, flags, cpu) \
+ ({ \
+ (void)cpu; \
+ local_irq_save(flags); \
+ &__get_cpu_var(var); \
+ })
+
+#define swap_put_cpu_var_irq_restore(var, flags, cpu) \
+ local_irq_restore(flags)
+
+#define swap_get_cpu_var(var, cpu) \
+ ({ \
+ (void)cpu; \
+ &get_cpu_var(var); \
+ })
+
+#define swap_put_cpu_var(var, cpu) put_cpu_var(var)
+
+#define swap_per_cpu_lock(var, cpu) &per_cpu(var, cpu)
+
+#define swap_per_cpu_unlock(var, cpu) do { } while (0)
+
+#define swap_get_cpu() get_cpu()
+
+#define swap_put_cpu() put_cpu()
+
+#define swap_irq_save(flags) local_irq_save(flags)
+
+#define swap_irq_restore(flags) local_irq_restore(flags)
+
+#endif
+
/*
* This path almost never happens for VM activity - pages are normally
* freed via pagevecs. But it gets used by networking.
@@ -142,13 +219,13 @@ void rotate_reclaimable_page(struct page *page)
!PageUnevictable(page) && PageLRU(page)) {
struct pagevec *pvec;
unsigned long flags;
+ int cpu;
page_cache_get(page);
- local_irq_save(flags);
- pvec = &__get_cpu_var(lru_rotate_pvecs);
+ pvec = swap_get_cpu_var_irq_save(lru_rotate_pvecs, flags, cpu);
if (!pagevec_add(pvec, page))
pagevec_move_tail(pvec);
- local_irq_restore(flags);
+ swap_put_cpu_var_irq_restore(lru_rotate_pvecs, flags, cpu);
}
}
@@ -217,12 +294,14 @@ EXPORT_SYMBOL(mark_page_accessed);
void __lru_cache_add(struct page *page, enum lru_list lru)
{
- struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
+ struct pagevec *pvec;
+ int cpu;
+ pvec = swap_get_cpu_var(lru_add_pvecs, cpu)[lru];
page_cache_get(page);
if (!pagevec_add(pvec, page))
____pagevec_lru_add(pvec, lru);
- put_cpu_var(lru_add_pvecs);
+ swap_put_cpu_var(lru_add_pvecs, cpu);
}
/**
@@ -272,31 +351,33 @@ void add_page_to_unevictable_list(struct page *page)
*/
static void drain_cpu_pagevecs(int cpu)
{
- struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
- struct pagevec *pvec;
+ struct pagevec *pvecs, *pvec;
int lru;
+ pvecs = swap_per_cpu_lock(lru_add_pvecs, cpu)[0];
for_each_lru(lru) {
pvec = &pvecs[lru - LRU_BASE];
if (pagevec_count(pvec))
____pagevec_lru_add(pvec, lru);
}
+ swap_per_cpu_unlock(lru_add_pvecs, cpu);
- pvec = &per_cpu(lru_rotate_pvecs, cpu);
+ pvec = swap_per_cpu_lock(lru_rotate_pvecs, cpu);
if (pagevec_count(pvec)) {
unsigned long flags;
/* No harm done if a racing interrupt already did this */
- local_irq_save(flags);
+ swap_irq_save(flags);
pagevec_move_tail(pvec);
- local_irq_restore(flags);
+ swap_irq_restore(flags);
}
+ swap_per_cpu_unlock(lru_rotate_pvecs, cpu);
}
void lru_add_drain(void)
{
- drain_cpu_pagevecs(get_cpu());
- put_cpu();
+ drain_cpu_pagevecs(swap_get_cpu());
+ swap_put_cpu();
}
static void lru_add_drain_per_cpu(struct work_struct *dummy)
@@ -370,7 +451,7 @@ void release_pages(struct page **pages, int nr, int cold)
}
__pagevec_free(&pages_to_free);
pagevec_reinit(&pages_to_free);
- }
+ }
}
if (zone)
spin_unlock_irqrestore(&zone->lru_lock, flags);
--
1.7.0.4
|
