diff options
author | Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | 2024-03-01 19:38:07 -0500 |
---|---|---|
committer | Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | 2024-03-01 20:32:11 -0500 |
commit | 18f21b27f01b9147cd95c538ece07f95c37150a2 (patch) | |
tree | 2d5422d267589726bb1d07a3c353ccdbcafe284e | |
parent | 3eff770448484493fa15ea90db643f86c124db6c (diff) | |
download | librseq-18f21b27f01b9147cd95c538ece07f95c37150a2.tar.gz |
x86: Introduce rseq_load_cbne_load_add_store
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: Id5d8ebfe056ef60ba38a7ab0bf1e368ca324ab25
-rw-r--r-- | include/rseq/arch/x86/bits.h | 111 |
1 files changed, 111 insertions, 0 deletions
diff --git a/include/rseq/arch/x86/bits.h b/include/rseq/arch/x86/bits.h index ab6313f..09fd49f 100644 --- a/include/rseq/arch/x86/bits.h +++ b/include/rseq/arch/x86/bits.h @@ -79,6 +79,61 @@ error2: #endif } +#define rseq_arch_has_load_cbne_load_add_store +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_add_store__ptr)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpq %[v], %[expect]\n\t" + "jne %l[ne]\n\t" + RSEQ_INJECT_ASM(4) + /* final store */ + "addq %[count], %[v2]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [v2] "m" (*v2), + [count] "er" (count) + : "memory", "cc", "rax" + RSEQ_INJECT_CLOBBER + : abort, ne +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +ne: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, int cpu) @@ -589,6 +644,62 @@ error2: #endif } +#define rseq_arch_has_load_cbne_load_add_store +static inline __attribute__((always_inline)) +int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_add_store__ptr)(intptr_t *v, intptr_t expect, + intptr_t *v2, intptr_t count, int cpu) +{ + RSEQ_INJECT_C(9) + + __asm__ __volatile__ goto ( + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne]) +#ifdef RSEQ_COMPARE_TWICE + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) +#endif + /* Start rseq by storing table entry pointer into rseq_cs. */ + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_ASM_CS_OFFSET(%[rseq_offset])) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_INDEX_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_INJECT_ASM(3) + "cmpl %[v], %[expect]\n\t" + "jne %l[ne]\n\t" + RSEQ_INJECT_ASM(4) + /* final store */ + "addl %[count], %[v2]\n\t" + "2:\n\t" + RSEQ_INJECT_ASM(5) + RSEQ_ASM_DEFINE_ABORT(4, "", abort) + : /* gcc asm goto does not allow outputs */ + : [cpu_id] "r" (cpu), + [rseq_offset] "r" (rseq_offset), + [v] "m" (*v), + [expect] "r" (expect), + [v2] "m" (*v2), + [count] "ir" (count) + : "memory", "cc", "eax" + RSEQ_INJECT_CLOBBER + : abort, ne +#ifdef RSEQ_COMPARE_TWICE + , error1 +#endif + ); + rseq_after_asm_goto(); + return 0; +abort: + rseq_after_asm_goto(); + RSEQ_INJECT_FAILED + return -1; +ne: + rseq_after_asm_goto(); + return 1; +#ifdef RSEQ_COMPARE_TWICE +error1: + rseq_after_asm_goto(); + rseq_bug("cpu_id comparison failed"); +#endif +} + + static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, int cpu) |