1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
|
/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
/* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/*
* rseq-riscv.h
*/
/*
* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
* are not part of the public API.
*/
#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/riscv.h> directly; include <rseq/rseq.h> instead."
#endif
/*
* Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
* other architectures, the ebreak instruction has no immediate field for
* distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
* "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
* is an uncommon instruction and will raise an illegal instruction
* exception when executed in all modes.
*/
#include <endian.h>
#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */
#else
#error "Currently, RSEQ only supports Little-Endian version"
#endif
/*
* Instruction selection between 32-bit/64-bit. Used internally in the
* rseq headers.
*/
#if __riscv_xlen == 64
#define __RSEQ_ASM_REG_SEL(a, b) a
#elif __riscv_xlen == 32
#define __RSEQ_ASM_REG_SEL(a, b) b
#endif
#define RSEQ_ASM_REG_L __RSEQ_ASM_REG_SEL("ld ", "lw ")
#define RSEQ_ASM_REG_S __RSEQ_ASM_REG_SEL("sd ", "sw ")
/*
* Refer to the Linux kernel memory model (LKMM) for documentation of
* the memory barriers.
*/
/* Only used internally in rseq headers. */
#define RSEQ_ASM_RISCV_FENCE(p, s) \
__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
/* CPU memory barrier. */
#define rseq_smp_mb() RSEQ_ASM_RISCV_FENCE(rw, rw)
/* CPU read memory barrier */
#define rseq_smp_rmb() RSEQ_ASM_RISCV_FENCE(r, r)
/* CPU write memory barrier */
#define rseq_smp_wmb() RSEQ_ASM_RISCV_FENCE(w, w)
/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p) \
__extension__ ({ \
rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
RSEQ_ASM_RISCV_FENCE(r, rw); \
____p1; \
})
/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v) \
do { \
RSEQ_ASM_RISCV_FENCE(rw, w); \
RSEQ_WRITE_ONCE(*(p), v); \
} while (0)
/* Temporary registers. */
#define RSEQ_ASM_TMP_REG_1 "t6"
#define RSEQ_ASM_TMP_REG_2 "t5"
#define RSEQ_ASM_TMP_REG_3 "t4"
#define RSEQ_ASM_TMP_REG_4 "t3"
/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
post_commit_offset, abort_ip) \
".pushsection __rseq_cs, \"aw\"\n" \
".balign 32\n" \
__rseq_str(label) ":\n" \
".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
".quad " __rseq_str(start_ip) ", " \
__rseq_str(post_commit_offset) ", " \
__rseq_str(abort_ip) "\n" \
".popsection\n\t" \
".pushsection __rseq_cs_ptr_array, \"aw\"\n" \
".quad " __rseq_str(label) "b\n" \
".popsection\n"
/*
* Define an rseq critical section structure of version 0 with no flags.
*
* @label:
* Local label for the beginning of the critical section descriptor
* structure.
* @start_ip:
* Pointer to the first instruction of the sequence of consecutive assembly
* instructions.
* @post_commit_ip:
* Pointer to the instruction after the last instruction of the sequence of
* consecutive assembly instructions.
* @abort_ip:
* Pointer to the instruction where to move the execution flow in case of
* abort of the sequence of consecutive assembly instructions.
*/
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
(post_commit_ip) - (start_ip), abort_ip)
/*
* Define the @exit_ip pointer as an exit point for the sequence of consecutive
* assembly instructions at @start_ip.
*
* @start_ip:
* Pointer to the first instruction of the sequence of consecutive assembly
* instructions.
* @exit_ip:
* Pointer to an exit point instruction.
*
* Exit points of a rseq critical section consist of all instructions outside
* of the critical section where a critical section can either branch to or
* reach through the normal course of its execution. The abort IP and the
* post-commit IP are already part of the __rseq_cs section and should not be
* explicitly defined as additional exit points. Knowing all exit points is
* useful to assist debuggers stepping over the critical section.
*/
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
".pushsection __rseq_exit_point_array, \"aw\"\n" \
".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
".popsection\n"
/*
* Define a critical section abort handler.
*
* @label:
* Local label to the abort handler.
* @teardown:
* Sequence of instructions to run on abort.
* @abort_label:
* C label to jump to at the end of the sequence.
*/
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
"j 222f\n" \
".balign 4\n" \
".long " __rseq_str(RSEQ_SIG) "\n" \
__rseq_str(label) ":\n" \
teardown \
"j %l[" __rseq_str(abort_label) "]\n" \
"222:\n"
/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
RSEQ_INJECT_ASM(1) \
"la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
__rseq_str(label) ":\n"
/* Store @value to address @var. */
#define RSEQ_ASM_OP_STORE(value, var) \
RSEQ_ASM_REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
/* Jump to local label @label when @var != @expect. */
#define RSEQ_ASM_OP_CBNE(var, expect, label) \
RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
__rseq_str(label) "\n"
/*
* Jump to local label @label when @var != @expect (32-bit register
* comparison).
*/
#define RSEQ_ASM_OP_CBNE32(var, expect, label) \
"lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
__rseq_str(label) "\n"
/* Jump to local label @label when @var == @expect. */
#define RSEQ_ASM_OP_CBEQ(var, expect, label) \
RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
"beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
__rseq_str(label) "\n"
/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
RSEQ_INJECT_ASM(2) \
RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
/* Load @var into temporary register. */
#define RSEQ_ASM_OP_R_LOAD(var) \
RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
/* Store from temporary register into @var. */
#define RSEQ_ASM_OP_R_STORE(var) \
RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
/* Load from address in temporary register+@offset into temporary register. */
#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
"add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \
RSEQ_ASM_TMP_REG_1 "\n" \
RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
/* Add @count to temporary register. */
#define RSEQ_ASM_OP_R_ADD(count) \
"add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \
", %[" __rseq_str(count) "]\n"
/*
* End-of-sequence store of @value to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
RSEQ_ASM_OP_STORE(value, var) \
__rseq_str(post_commit_label) ":\n"
/*
* End-of-sequence store-release of @value to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
"fence rw, w\n" \
RSEQ_ASM_OP_STORE(value, var) \
__rseq_str(post_commit_label) ":\n"
/*
* End-of-sequence store of temporary register to address @var. Emit
* @post_commit_label label after the store instruction.
*/
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
__rseq_str(post_commit_label) ":\n"
/*
* Copy @len bytes from @src to @dst. This is an inefficient bytewise
* copy and could be improved in the future.
*/
#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len) \
"beqz %[" __rseq_str(len) "], 333f\n" \
"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \
"mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \
"mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \
"222:\n" \
"lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \
"sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \
"addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \
"addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \
"addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \
"bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \
"333:\n"
/*
* Load pointer address from @ptr. Add @off to offset from this pointer.
* Add @inc to the resulting address as an end-of-sequence store.
*/
#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, post_commit_label) \
"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \
RSEQ_ASM_OP_R_ADD(off) \
RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
RSEQ_ASM_OP_R_ADD(inc) \
__rseq_str(post_commit_label) ":\n"
/* Per-cpu-id indexing. */
#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID
/* Per-mm-cid indexing. */
#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID
/* APIs which are not indexed. */
#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
|
