1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
|
/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
/*
* rseq/arch/aarch64.h
*/
#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/aarch64.h> directly; include <rseq/rseq.h> instead."
#endif
/*
* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
* are not part of the public API.
*/
/*
* aarch64 -mbig-endian generates mixed endianness code vs data:
* little-endian code and big-endian data. Ensure the RSEQ_SIG signature
* matches code endianness.
*/
#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
#ifdef __AARCH64EB__ /* Big endian */
# define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
#else /* Little endian */
# define RSEQ_SIG_DATA RSEQ_SIG_CODE
#endif
#define RSEQ_SIG RSEQ_SIG_DATA
/*
* Refer to the Linux kernel memory model (LKMM) for documentation of
* the memory barriers.
*/
/* CPU memory barrier. */
#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
/* CPU read memory barrier */
#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
/* CPU write memory barrier */
#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p) \
__extension__ ({ \
union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
switch (sizeof(*(p))) { \
case 1: \
__asm__ __volatile__ ("ldarb %w0, %1" \
: "=r" (*(__u8 *)__u.__c) \
: "Q" (*(p)) : "memory"); \
break; \
case 2: \
__asm__ __volatile__ ("ldarh %w0, %1" \
: "=r" (*(__u16 *)__u.__c) \
: "Q" (*(p)) : "memory"); \
break; \
case 4: \
__asm__ __volatile__ ("ldar %w0, %1" \
: "=r" (*(__u32 *)__u.__c) \
: "Q" (*(p)) : "memory"); \
break; \
case 8: \
__asm__ __volatile__ ("ldar %0, %1" \
: "=r" (*(__u64 *)__u.__c) \
: "Q" (*(p)) : "memory"); \
break; \
} \
(rseq_unqual_scalar_typeof(*(p)))__u.__val; \
})
/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v) \
do { \
union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
{ .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
switch (sizeof(*(p))) { \
case 1: \
__asm__ __volatile__ ("stlrb %w1, %0" \
: "=Q" (*(p)) \
: "r" (*(__u8 *)__u.__c) \
: "memory"); \
break; \
case 2: \
__asm__ __volatile__ ("stlrh %w1, %0" \
: "=Q" (*(p)) \
: "r" (*(__u16 *)__u.__c) \
: "memory"); \
break; \
case 4: \
__asm__ __volatile__ ("stlr %w1, %0" \
: "=Q" (*(p)) \
: "r" (*(__u32 *)__u.__c) \
: "memory"); \
break; \
case 8: \
__asm__ __volatile__ ("stlr %1, %0" \
: "=Q" (*(p)) \
: "r" (*(__u64 *)__u.__c) \
: "memory"); \
break; \
} \
} while (0)
#define RSEQ_ASM_U64_PTR(x) ".quad " x
#define RSEQ_ASM_U32(x) ".long " x
/* Temporary scratch registers. */
#define RSEQ_ASM_TMP_REG32 "w15"
#define RSEQ_ASM_TMP_REG "x15"
#define RSEQ_ASM_TMP_REG_2 "x14"
/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
post_commit_offset, abort_ip) \
" .pushsection __rseq_cs, \"aw\"\n" \
" .balign 32\n" \
__rseq_str(label) ":\n" \
" " RSEQ_ASM_U32(__rseq_str(version)) "\n" \
" " RSEQ_ASM_U32(__rseq_str(flags)) "\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n" \
" .popsection\n\t" \
" .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n" \
" .popsection\n"
/*
* Define an rseq critical section structure of version 0 with no flags.
*
* @label:
* Local label for the beginning of the critical section descriptor
* structure.
* @start_ip:
* Pointer to the first instruction of the sequence of consecutive assembly
* instructions.
* @post_commit_ip:
* Pointer to the instruction after the last instruction of the sequence of
* consecutive assembly instructions.
* @abort_ip:
* Pointer to the instruction where to move the execution flow in case of
* abort of the sequence of consecutive assembly instructions.
*/
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
(post_commit_ip) - (start_ip), abort_ip)
/*
* Define the @exit_ip pointer as an exit point for the sequence of consecutive
* assembly instructions at @start_ip.
*
* @start_ip:
* Pointer to the first instruction of the sequence of consecutive assembly
* instructions.
* @exit_ip:
* Pointer to an exit point instruction.
*
* Exit points of a rseq critical section consist of all instructions outside
* of the critical section where a critical section can either branch to or
* reach through the normal course of its execution. The abort IP and the
* post-commit IP are already part of the __rseq_cs section and should not be
* explicitly defined as additional exit points. Knowing all exit points is
* useful to assist debuggers stepping over the critical section.
*/
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
" .pushsection __rseq_exit_point_array, \"aw\"\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
" " RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n" \
" .popsection\n"
/*
* Define a critical section abort handler.
*
* @label:
* Local label to the abort handler.
* @teardown:
* Sequence of instructions to run on abort.
* @abort_label:
* C label to jump to at the end of the sequence.
*/
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
" b 222f\n" \
" .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
__rseq_str(label) ":\n" \
teardown \
" b %l[" __rseq_str(abort_label) "]\n" \
"222:\n"
/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
RSEQ_INJECT_ASM(1) \
" adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
", :lo12:" __rseq_str(cs_label) "\n" \
" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
__rseq_str(label) ":\n"
/* Store @value to address @var. */
#define RSEQ_ASM_OP_STORE(value, var) \
" str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
/* Store-release @value to address @var. */
#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
" stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
/*
* End-of-sequence store of @value to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
RSEQ_ASM_OP_STORE(value, var) \
__rseq_str(post_commit_label) ":\n"
/*
* End-of-sequence store-release of @value to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
RSEQ_ASM_OP_STORE_RELEASE(value, var) \
__rseq_str(post_commit_label) ":\n"
/* Jump to local label @label when @var != @expect. */
#define RSEQ_ASM_OP_CBNE(var, expect, label) \
" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
", %[" __rseq_str(expect) "]\n" \
" cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
/*
* Jump to local label @label when @var != @expect (32-bit register
* comparison).
*/
#define RSEQ_ASM_OP_CBNE32(var, expect, label) \
" ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
" sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
", %w[" __rseq_str(expect) "]\n" \
" cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
/* Jump to local label @label when @var == @expect. */
#define RSEQ_ASM_OP_CBEQ(var, expect, label) \
" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
", %[" __rseq_str(expect) "]\n" \
" cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
RSEQ_INJECT_ASM(2) \
RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
/* Load @var into temporary register. */
#define RSEQ_ASM_OP_R_LOAD(var) \
" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
/* Store from temporary register into @var. */
#define RSEQ_ASM_OP_R_STORE(var) \
" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
/* Load from address in temporary register+@offset into temporary register. */
#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
" ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
", %[" __rseq_str(offset) "]]\n"
/* Add @count to temporary register. */
#define RSEQ_ASM_OP_R_ADD(count) \
" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
", %[" __rseq_str(count) "]\n"
/*
* End-of-sequence store of temporary register to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
__rseq_str(post_commit_label) ":\n"
/*
* Copy @len bytes from @src to @dst. This is an inefficient bytewise
* copy and could be improved in the future.
*/
#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len) \
" cbz %[" __rseq_str(len) "], 333f\n" \
" mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
"222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
" ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
", " RSEQ_ASM_TMP_REG_2 "]\n" \
" strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
", " RSEQ_ASM_TMP_REG_2 "]\n" \
" cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
"333:\n"
/* Per-cpu-id indexing. */
#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID
/* Per-mm-cid indexing. */
#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID
/* APIs which are not indexed. */
#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
|
