1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/*
* rseq/arch/x86.h
*/
#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/x86.h> directly; include <rseq/rseq.h> instead."
#endif
#include <stdint.h>
/*
* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
* are not part of the public API.
*/
/*
* RSEQ_SIG is used with the following reserved undefined instructions, which
* trap in user-space:
*
* x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
* x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
*/
#define RSEQ_SIG 0x53053053
/*
* Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
* operands, we cannot use "m" input operands, and rather pass the __rseq_abi
* address through a "r" input operand.
*/
/*
* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
* are defined explicitly as macros to be used from assembly.
*/
#define RSEQ_ASM_CPU_ID_OFFSET 4
#define RSEQ_ASM_CS_OFFSET 8
#define RSEQ_ASM_MM_CID_OFFSET 24
/*
* Refer to the Linux kernel memory model (LKMM) for documentation of
* the memory barriers. Expect all x86 hardware to be x86-TSO (Total
* Store Order).
*/
/* CPU memory barrier. */
#define rseq_smp_mb() \
__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
/* CPU read memory barrier */
#define rseq_smp_rmb() rseq_barrier()
/* CPU write memory barrier */
#define rseq_smp_wmb() rseq_barrier()
/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p) \
__extension__ ({ \
rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
rseq_barrier(); \
____p1; \
})
/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v) \
do { \
rseq_barrier(); \
RSEQ_WRITE_ONCE(*(p), v); \
} while (0)
/* Segment selector for the thread pointer. */
#ifdef RSEQ_ARCH_AMD64
# define RSEQ_ASM_TP_SEGMENT %%fs
#else
# define RSEQ_ASM_TP_SEGMENT %%gs
#endif
/*
* Helper macro to define a variable of pointer type stored in a 64-bit
* integer. Only used internally in rseq headers.
*/
#ifdef RSEQ_ARCH_AMD64
# define RSEQ_ASM_U64_PTR(x) ".quad " x
#else
# define RSEQ_ASM_U64_PTR(x) ".long " x ", 0x0"
#endif
#define RSEQ_ASM_U32(x) ".long " x
/* Common architecture support macros. */
#include "rseq/arch/generic/common.h"
/*
* Define a critical section abort handler.
*
* @label:
* Local label to the abort handler.
* @teardown:
* Sequence of instructions to run on abort.
* @abort_label:
* C label to jump to at the end of the sequence.
*/
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
".pushsection __rseq_failure, \"ax\"\n\t" \
/* \
* Disassembler-friendly signature: \
* x86-32: ud1 <sig>,%edi \
* x86-64: ud1 <sig>(%rip),%edi \
*/ \
".byte 0x0f, 0xb9, 0x3d\n\t" \
".long " __rseq_str(RSEQ_SIG) "\n\t" \
__rseq_str(label) ":\n\t" \
teardown \
"jmp %l[" __rseq_str(abort_label) "]\n\t" \
".popsection\n\t"
/*
* Define a critical section teardown handler.
*
* @label:
* Local label to the teardown handler.
* @teardown:
* Sequence of instructions to run on teardown.
* @target_label:
* C label to jump to at the end of the sequence.
*/
#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
".pushsection __rseq_failure, \"ax\"\n\t" \
__rseq_str(label) ":\n\t" \
teardown \
"jmp %l[" __rseq_str(target_label) "]\n\t" \
".popsection\n\t"
/*
* Store the address of the critical section descriptor structure at
* @cs_label into the @rseq_cs pointer and emit the label @label, which
* is the beginning of the sequence of consecutive assembly instructions.
*
* @label:
* Local label to the beginning of the sequence of consecutive assembly
* instructions.
* @cs_label:
* Source local label to the critical section descriptor structure.
* @rseq_cs:
* Destination pointer where to store the address of the critical
* section descriptor structure.
*/
#ifdef RSEQ_ARCH_AMD64
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
RSEQ_INJECT_ASM(1) \
"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
"movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
__rseq_str(label) ":\n\t"
#else
/*
* Use ip-relative addressing to get the address to the rseq critical
* section descriptor. On x86-32, this requires a "call" instruction to
* get the instruction pointer, which modifies the stack. Beware of this
* side-effect if this scheme is used within a rseq critical section.
* This computation is performed immediately before storing the rseq_cs,
* which is outside of the critical section.
* Balance call/ret to help speculation.
*/
# define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
RSEQ_INJECT_ASM(1) \
"call 880f\n\t" \
"880:\n\t" \
"popl %%eax\n\t" \
"leal (881f-880b)(%%eax), %%eax\n\t" \
"pushl %%eax\n\t" \
"ret\n\t" \
"881:\n\t" \
"leal (" __rseq_str(cs_label) " - 881b)(%%eax), %%eax\n\t" \
"movl %%eax, " __rseq_str(rseq_cs) "\n\t" \
__rseq_str(label) ":\n\t"
#endif
/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
RSEQ_INJECT_ASM(2) \
"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
"jnz " __rseq_str(label) "\n\t"
/* Per-cpu-id indexing. */
#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID
/* Per-mm-cid indexing. */
#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID
/* APIs which are not indexed. */
#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
|
