x86-64: Fix call instruction caller-save register clobbering

Given this test program:

    public class Hashtable<K, V> {
      private final float loadFactor;

      public Hashtable(int initialCapacity, float loadFactor)
      {
        HashEntry<K, V>[] buckets = (HashEntry<K, V>[]) new HashEntry[initialCapacity];
        if (!(loadFactor > 0))
          throw new IllegalArgumentException();
        this.loadFactor = loadFactor;
      }

      private static final class HashEntry<K, V> {
      }

      public static void main(String[] args) {
        Hashtable ht = new Hashtable(100, 0.75f);
      }
    }

If you compile Jato with GCC optimization level "-O2", for example:

    make CFLAGS=-O2

and then run the program as follows:

    javac Hashtable.java

    ./jato -Xtrace:exceptions \
       -bootclasspath .:/usr/local/classpath/share/classpath/glibj.zip \
       -Djava.library.path=/usr/local/classpath/lib/classpath/ \
       -Xnosystemclassloader Hashtable

You will see the following crash:

    [main] trace exception: exception object 0x236c630 (java/lang/IllegalArgumentException) thrown
    [main] 	message	: 0x0 null
    [main] 	from	: 0x40fa8fc7: Hashtable.<init>(IF)V
    [main] 		  (Hashtable.java:12)
    [main] 	action	: unwind to 0x40fa8db1: Hashtable.main([Ljava/lang/String;)V
    [main] 		  (Hashtable.java:20)
    [main] trace exception: exception object 0x236c630 (java/lang/IllegalArgumentException) thrown
    [main] 	message	: 0x0 null
    [main] 	from	: 0x40fa8db0: Hashtable.main([Ljava/lang/String;)V
    [main] 		  (Hashtable.java:20)
    [main] 	action	: unwind to native caller at 0x41e4b8
  Exception in thread "main" java.lang.IllegalArgumentException
     at Hashtable.<init>(Hashtable.java:12)
     at Hashtable.main(Hashtable.java:20)

Now while this doesn't seem to happen with the "-Os" GCC optimization
level, it's by pure luck. Looking at the generated assembly, we see that
XMM registers (which are *caller saved* registers on x86-64) are not
saved before call instructions:

    [main]   0x41d7ff51:   48 89 e5                   mov    %rsp,%rbp
    [main]   0x41d7ff54:   48 83 ec 40                sub    $0x40,%rsp
    [main]   0x41d7ff58:   53                         push   %rbx
    [main]   0x41d7ff59:   41 54                      push   %r12
    [main]   0x41d7ff5b:   41 55                      push   %r13
    [main]   0x41d7ff5d:   41 56                      push   %r14
    [main]   0x41d7ff5f:   41 57                      push   %r15
    [main]   0x41d7ff61:   57                         push   %rdi
    [main]   0x41d7ff62:   49 89 ff                   mov    %rdi,%r15
    [main]   0x41d7ff65:   49 89 f6                   mov    %rsi,%r14
    [main]   0x41d7ff68:   f3 0f 10 f8                movss  %xmm0,%xmm7
    [main]   0x41d7ff6c:   4d 89 fd                   mov    %r15,%r13
    [main]   0x41d7ff6f:   4d 85 6d 00                test   %r13,0x0(%r13)
    [main]   0x41d7ff73:   4c 89 ef                   mov    %r13,%rdi
    [main]   0x41d7ff76:   f6 04 25 00 90 d3 00 00    testb  $0x0,0xd39000
    [main]   0x41d7ff7e:   e8 bd cd ff ff             callq  0x0000000041d7cd40       # java/lang/Object.<init>()V
    [main]   0x41d7ff83:   45 89 f6                   mov    %r14d,%r14d
    [main]   0x41d7ff86:   4c 89 f7                   mov    %r14,%rdi
    [main]   0x41d7ff89:   e8 c2 51 6b be             callq  0x0000000000435150       # array_size_check+0 (/home/penberg/jato/vm/object.c:494)

which means that the any function that uses the XMM registers will
clobber our registers.

To fix the issue, use the following simple strategy suggested by "Linear
Scan Register Allocation" by Poletto and Sarkar (1999):

  The simplest solution is to use all registers, and insert saves and
  restores where appropriate around function calls after register
  allocation.

This fixes #25.

Signed-off-by: Pekka Enberg <penberg@kernel.org>

16 files changed, 369 insertions, 7 deletions

diff --git a/Makefile b/Makefile
index 7807bfec..25162a00 100644
--- a/Makefile
+++ b/Makefile
@@ -56,6 +56,7 @@ LIB_OBJS += jit/bc-offset-mapping.o
 LIB_OBJS += jit/branch-bc.o
 LIB_OBJS += jit/bytecode-to-ir.o
 LIB_OBJS += jit/cfg-analyzer.o
+LIB_OBJS += jit/clobber.o
 LIB_OBJS += jit/compilation-unit.o
 LIB_OBJS += jit/compiler.o
 LIB_OBJS += jit/constant-pool.o
diff --git a/arch/x86/emit_32.c b/arch/x86/emit_32.c
index 42a30832..8d69e0b7 100644
--- a/arch/x86/emit_32.c
+++ b/arch/x86/emit_32.c
@@ -1196,6 +1196,9 @@ void *emit_itable_resolver_stub(struct vm_class *vmc,
 	return buffer_ptr(buf);
 }
 
+static void emit_pseudo(struct insn *insn, struct buffer *buffer, struct basic_block *bb)
+{
+}
 
 typedef void (*emit_fn_t)(struct insn *insn, struct buffer *, struct basic_block *bb);
 
@@ -1325,6 +1328,12 @@ static emit_fn_t emitters[] = {
 	DECL_EMITTER(INSN_SUB_MEMBASE_REG, insn_encode),
 	DECL_EMITTER(INSN_TEST_IMM_MEMDISP, emit_test_imm_memdisp),
 	DECL_EMITTER(INSN_TEST_MEMBASE_REG, insn_encode),
+	DECL_EMITTER(INSN_SAVE_CALLER_REGS, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_I32, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_I64, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_F32, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_F64, emit_pseudo),
 };
 
 static void __emit_insn(struct buffer *buf, struct basic_block *bb, struct insn *insn)
diff --git a/arch/x86/emit_64.c b/arch/x86/emit_64.c
index f3daefd0..d240257b 100644
--- a/arch/x86/emit_64.c
+++ b/arch/x86/emit_64.c
@@ -1610,6 +1610,9 @@ void *emit_itable_resolver_stub(struct vm_class *vmc,
 	return buffer_ptr(buf);
 }
 
+static void emit_pseudo(struct insn *insn, struct buffer *buffer, struct basic_block *bb)
+{
+}
 
 typedef void (*emit_fn_t)(struct insn *insn, struct buffer *, struct basic_block *bb);
 
@@ -1710,6 +1713,12 @@ static emit_fn_t emitters[] = {
 	DECL_EMITTER(INSN_PUSH_IMM, emit_push_imm),
 	DECL_EMITTER(INSN_TEST_MEMBASE_REG, emit_test_membase_reg),
 	DECL_EMITTER(INSN_TEST_IMM_MEMDISP, emit_test_imm_memdisp),
+	DECL_EMITTER(INSN_SAVE_CALLER_REGS, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_I32, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_I64, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_F32, emit_pseudo),
+	DECL_EMITTER(INSN_RESTORE_CALLER_REGS_F64, emit_pseudo),
 };
 
 static void __emit_insn(struct buffer *buf, struct basic_block *bb, struct insn *insn)
diff --git a/arch/x86/include/arch/instruction.h b/arch/x86/include/arch/instruction.h
index 907d33ea..54c97806 100644
--- a/arch/x86/include/arch/instruction.h
+++ b/arch/x86/include/arch/instruction.h
@@ -215,6 +215,13 @@ enum insn_type {
 	INSN_XOR_REG_REG,
 	INSN_XORPS_XMM_XMM,
 
+	INSN_SAVE_CALLER_REGS,
+	INSN_RESTORE_CALLER_REGS,
+	INSN_RESTORE_CALLER_REGS_I32,
+	INSN_RESTORE_CALLER_REGS_I64,
+	INSN_RESTORE_CALLER_REGS_F32,
+	INSN_RESTORE_CALLER_REGS_F64,
+
 	/* Must be last */
 	NR_INSN_TYPES,
 };
diff --git a/arch/x86/include/arch/registers_32.h b/arch/x86/include/arch/registers_32.h
index ef6ffc8d..93722c2b 100644
--- a/arch/x86/include/arch/registers_32.h
+++ b/arch/x86/include/arch/registers_32.h
@@ -59,6 +59,23 @@ extern enum machine_reg caller_save_regs[NR_CALLER_SAVE_REGS];
 #define NR_CALLEE_SAVE_REGS 3
 extern enum machine_reg callee_save_regs[NR_CALLEE_SAVE_REGS];
 
+static inline bool is_return_reg(enum machine_reg reg, enum vm_type type)
+{
+	switch (type) {
+	case J_INT:
+		return reg == MACH_REG_EAX;
+	case J_LONG:
+		return reg == MACH_REG_EAX || reg == MACH_REG_EDX;
+	case J_FLOAT:
+		return reg == MACH_REG_XMM0;
+	case J_DOUBLE:
+		return reg == MACH_REG_XMM0;
+	default:
+		break;
+	}
+	return false;
+}
+
 const char *reg_name(enum machine_reg reg);
 
 extern uint32_t reg_type_table[NR_REGISTERS];
diff --git a/arch/x86/include/arch/registers_64.h b/arch/x86/include/arch/registers_64.h
index 789f13c2..ded7ef2d 100644
--- a/arch/x86/include/arch/registers_64.h
+++ b/arch/x86/include/arch/registers_64.h
@@ -81,6 +81,23 @@ extern enum machine_reg	arg_gp_regs[NR_ARG_GP_REGS];
 #define NR_ARG_XMM_REGS		8
 extern enum machine_reg	arg_xmm_regs[NR_ARG_XMM_REGS];
 
+static inline bool is_return_reg(enum machine_reg reg, enum vm_type type)
+{
+	switch (type) {
+	case J_INT:
+		return reg == MACH_REG_RAX;
+	case J_LONG:
+		return reg == MACH_REG_RAX;
+	case J_FLOAT:
+		return reg == MACH_REG_XMM0;
+	case J_DOUBLE:
+		return reg == MACH_REG_XMM0;
+	default:
+		break;
+	}
+	return false;
+}
+
 const char *reg_name(enum machine_reg reg);
 enum machine_reg_type reg_type(enum machine_reg reg);
 bool reg_supports_type(enum machine_reg reg, enum vm_type type);
diff --git a/arch/x86/insn-selector_64.brg b/arch/x86/insn-selector_64.brg
index ebb95c77..9f09868e 100644
--- a/arch/x86/insn-selector_64.brg
+++ b/arch/x86/insn-selector_64.brg
@@ -445,9 +445,11 @@ freg:	OP_DREM(freg, freg) 1
 	xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
 	xmm1 = get_fixed_var(s->b_parent, MACH_REG_XMM1);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, state->right->reg1, xmm1));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, state->left->reg1, xmm0));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long)fmod));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_F64));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, xmm0, state->reg1));
 }
 
@@ -460,9 +462,11 @@ freg:	OP_FREM(freg, freg) 1
 	xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
 	xmm1 = get_fixed_var(s->b_parent, MACH_REG_XMM1);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->right->reg1, xmm1));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->left->reg1, xmm0));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long)fmodf));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_F32));
 	select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, xmm0, state->reg1));
 }
 
@@ -598,6 +602,8 @@ reg:	OP_CMPL(freg, freg) 1
 	arg1 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
 	arg2 = get_fixed_var(s->b_parent, MACH_REG_XMM1);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	if (type == J_FLOAT)
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->left->reg1, arg1));
 	else
@@ -613,6 +619,8 @@ reg:	OP_CMPL(freg, freg) 1
 	else
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_dcmpl));
 
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
+
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 }
 
@@ -630,6 +638,8 @@ reg:	OP_CMPG(freg, freg) 1
 	arg1 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
 	arg2 = get_fixed_var(s->b_parent, MACH_REG_XMM1);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	if (type == J_FLOAT)
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->left->reg1, arg1));
 	else
@@ -645,6 +655,8 @@ reg:	OP_CMPG(freg, freg) 1
 	else
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_dcmpg));
 
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
+
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 }
 
@@ -769,9 +781,12 @@ reg:	EXPR_CLASS_FIELD 1
 		struct var_info *rdi;
 
 		rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmc, rdi));
 		select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long)vm_class_ensure_init));
 
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
+
 		mov_insn = memdisp_reg_insn(INSN_MOV_MEMDISP_REG, (unsigned long) vmc->static_values + vmf->offset, out);
 	} else {
 		if (vmc_state >= VM_CLASS_INITIALIZING) {
@@ -818,8 +833,10 @@ freg:	EXPR_FLOAT_CLASS_FIELD 1
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmc->static_values + vmf->offset, offset));
 
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmc, rdi));
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long)vm_class_ensure_init));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 
 	if (expr->vm_type == J_FLOAT)
 		mov_insn = membase_reg_insn(INSN_MOVSS_MEMBASE_XMM, offset, 0, out);
@@ -848,8 +865,10 @@ reg:	EXPR_INSTANCE_FIELD(reg) 1
 
 		rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, state->left->reg1, rdi));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) &get_referent));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 
 		struct var_info *eax = get_fixed_var(s->b_parent, MACH_REG_xAX);
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, state->reg1));
@@ -887,8 +906,10 @@ reg:	EXPR_NEW
 
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) expr->class, rdi));
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_alloc));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 	select_exception_test(s, tree);
 }
@@ -902,8 +923,10 @@ reg:	EXPR_ARRAY_SIZE_CHECK(reg)
 
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, size, rdi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) array_size_check));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 	select_exception_test(s, tree);
 }
 
@@ -922,9 +945,11 @@ reg:	EXPR_NEWARRAY(reg)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, size, rsi));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, expr->array_type, rdi));
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_alloc_primitive_array));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 
 	select_exception_test(s, tree);
@@ -975,6 +1000,7 @@ reg:	EXPR_MULTIANEWARRAY(arg)
 	args_list = to_expr(expr->multianewarray_dimensions);
 	dimension = nr_args(args_list);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, dimension, rsi));
 
@@ -982,6 +1008,7 @@ reg:	EXPR_MULTIANEWARRAY(arg)
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) expr->multianewarray_ref_type, rdi));
 
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_alloc_multi_array));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, xax, state->reg1));
 
 	select_exception_test(s, tree);
@@ -1002,9 +1029,11 @@ reg:    EXPR_ANEWARRAY(reg)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) expr->anewarray_ref_type, rdi));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, size, rsi));
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_alloc_array));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 
 	select_exception_test(s, tree);
@@ -1038,9 +1067,11 @@ reg:	EXPR_INSTANCEOF(reg)
 
 	state->reg1 = get_var(s->b_parent, J_INT);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, ref, rdi));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) expr->instanceof_class, rsi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_is_instance_of));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 
 	select_exception_test(s, tree);
@@ -1160,8 +1191,10 @@ reg:	EXPR_CONVERSION_FROM_FLOAT(freg)
 
 		state->reg1 = get_var(s->b_parent, J_INT);
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->left->reg1, xmm0));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_f2i));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, state->reg1));
 	} else if (expr->vm_type == J_LONG) {
 		struct var_info *rax, *xmm0;
@@ -1171,8 +1204,10 @@ reg:	EXPR_CONVERSION_FROM_FLOAT(freg)
 
 		state->reg1 = get_var(s->b_parent, J_LONG);
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, state->left->reg1, xmm0));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_f2l));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 	} else {
 		die("EXPR_CONVERSION_FROM_FLOAT: no conversion from %d to %d", src->vm_type, expr->vm_type);
@@ -1196,8 +1231,10 @@ reg:	EXPR_CONVERSION_FROM_DOUBLE(freg)
 
 		state->reg1 = get_var(s->b_parent, J_INT);
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, state->left->reg1, xmm0));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_d2i));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, state->reg1));
 	} else if (expr->vm_type == J_LONG) {
 		struct var_info *rax, *xmm0;
@@ -1207,8 +1244,10 @@ reg:	EXPR_CONVERSION_FROM_DOUBLE(freg)
 
 		state->reg1 = get_var(s->b_parent, J_LONG);
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, state->left->reg1, xmm0));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) emulate_d2l));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 	} else {
 		die("EXPR_CONVERSION_FROM_DOUBLE: no conversion from %d to %d", src->vm_type, expr->vm_type);
@@ -1311,16 +1350,22 @@ reg:	EXPR_EXCEPTION_REF
 
 stmt:	STMT_INVOKE(arg) 1
 {
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	invoke(s, tree);
 }
 
 stmt:	STMT_INVOKEINTERFACE(arg) 1
 {
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	invokeinterface(state, s, tree);
 }
 
 stmt:	STMT_INVOKEVIRTUAL(arg) 1
 {
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	invokevirtual(state, s, tree);
 }
 
@@ -1434,6 +1479,8 @@ stmt:	STMT_STORE(EXPR_FLOAT_CLASS_FIELD, freg)
 	vmf = store_dest->class_field;
 	vmc = vmf->class;
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	/* FIXME: Use guard pages to fixup static accesses like on 32-bit. */
 
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmc->static_values + vmf->offset, offset));
@@ -1441,6 +1488,7 @@ stmt:	STMT_STORE(EXPR_FLOAT_CLASS_FIELD, freg)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmc, rdi));
 	select_safepoint_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long)vm_class_ensure_init));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 
 	if (store_dest->vm_type == J_FLOAT)
 		mov_insn = reg_membase_insn(INSN_MOVSS_XMM_MEMBASE, src, offset, 0);
@@ -1487,6 +1535,8 @@ stmt:	STMT_STORE(inst_field, reg)
 	if (vm_field_equals(store_dest->instance_field, vm_java_lang_ref_Reference_referent)) {
 		struct var_info *rdi, *rsi;
 
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 		rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, src, rsi));
 
@@ -1494,6 +1544,8 @@ stmt:	STMT_STORE(inst_field, reg)
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, base, rdi));
 
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) &put_referent));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
+
 		return;
 	}
 
@@ -1831,17 +1883,21 @@ stmt:	STMT_ARRAY_STORE_CHECK(reg, reg) 1
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
 	if (src_expr->vm_type == J_REFERENCE) {
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG,
 						  state->right->reg1, rdi));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG,
 						  state->left->reg1, rsi));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) array_store_check));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 	} else {
+		select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG,
 						  state->right->reg1, rdi));
 		select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG,
 						  src_expr->vm_type, rsi));
 		select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) array_store_check_vmtype));
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 	}
 
 	select_exception_test(s, tree);
@@ -1859,9 +1915,11 @@ stmt:	STMT_ARRAY_STORE_CHECK(freg, reg) 1
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, state->right->reg1, rdi));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, src_expr->vm_type, rsi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) array_store_check_vmtype));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 	select_exception_test(s, tree);
 }
 
@@ -1873,6 +1931,8 @@ stmt:	STMT_ATHROW(reg)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	select_insn(s, tree,
 		    imm_reg_insn(INSN_MOV_IMM_REG,
 				 (unsigned long) s->b_parent, rdi));
@@ -1880,6 +1940,7 @@ stmt:	STMT_ATHROW(reg)
 		    reg_reg_insn(INSN_MOV_REG_REG, state->left->reg1, rsi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL,
 				      (unsigned long) throw_exception));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
 
 	/* Jump where throw_exception() told us to jump */
 	select_insn(s, tree, reg_insn(INSN_PUSH_REG, rax));
@@ -1913,9 +1974,11 @@ stmt:	STMT_ARRAY_CHECK(array_check)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, ref, rdi));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, index, rsi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_check_array));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 
 	select_exception_test(s, tree);
 }
@@ -1973,6 +2036,8 @@ reg:	EXPR_LOOKUPSWITCH_BSEARCH(reg)
 
 	expr = to_expr(tree);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
+
 	r8 = get_fixed_var(s->b_parent, MACH_REG_R8);
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) &lookupswitch_pair_comp, r8));
 
@@ -1991,6 +2056,8 @@ reg:	EXPR_LOOKUPSWITCH_BSEARCH(reg)
 	rax = get_fixed_var(s->b_parent, MACH_REG_RAX);
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) &bsearch));
 
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
+
 	state->reg1 = get_var(s->b_parent, J_NATIVE_PTR);
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, state->reg1));
 }
@@ -2002,9 +2069,11 @@ stmt:	STMT_MONITOR_ENTER(reg)
 	ref = state->left->reg1;
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, ref, rdi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL,
 				      (unsigned long) vm_object_lock));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 
 	select_exception_test(s, tree);
 }
@@ -2016,9 +2085,11 @@ stmt:	STMT_MONITOR_EXIT(reg)
 	ref = state->left->reg1;
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, ref, rdi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL,
 				      (unsigned long) vm_object_unlock));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 
 	select_exception_test(s, tree);
 }
@@ -2035,9 +2106,11 @@ stmt:	STMT_CHECKCAST(reg)
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, ref, rdi));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) stmt->checkcast_class, rsi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) vm_object_check_cast));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 
 	select_exception_test(s, tree);
 }
@@ -2084,7 +2157,9 @@ static void select_exception_test(struct basic_block *bb,
 	struct var_info *reg;
 
 	if (running_on_valgrind) {
+		select_insn(bb, tree, insn(INSN_SAVE_CALLER_REGS));
 		select_insn(bb, tree, rel_insn(INSN_CALL_REL, (unsigned long) exception_check));
+		select_insn(bb, tree, insn(INSN_RESTORE_CALLER_REGS));
 		return;
 	}
 
@@ -2202,6 +2277,7 @@ emulate_op_64(struct _MBState *state, struct basic_block *s,
 
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, state->reg1));
 
+
 	select_exception_test(s, tree);
 }
 
@@ -2247,11 +2323,13 @@ static void select_trace_return_value(struct basic_block *s,
 	rdi = get_fixed_var(s->b_parent, MACH_REG_RDI);
 	rsi = get_fixed_var(s->b_parent, MACH_REG_RSI);
 
+	select_insn(s, tree, insn(INSN_SAVE_CALLER_REGS));
 	select_insn(s, tree, reg_insn(INSN_PUSH_REG, rax));
 	select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, rsi));
 	select_insn(s, tree, imm_reg_insn(INSN_MOV_IMM_REG, (unsigned long) vmm, rdi));
 	select_insn(s, tree, rel_insn(INSN_CALL_REL, (unsigned long) trace_return_value));
 	select_insn(s, tree, reverse_reg_insn(INSN_POP_REG, rax));
+	select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
 }
 
 static void save_invoke_result(struct basic_block *s,
@@ -2265,28 +2343,50 @@ static void save_invoke_result(struct basic_block *s,
 		select_trace_return_value(s, tree, method);
 
 	tmp = stmt->invoke_result;
-	if (!tmp)
+	if (!tmp) {
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS));
 		return;
+	}
 
 	switch (tmp->vm_type) {
 	case J_FLOAT: {
-		struct var_info *xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
+		struct var_info *xmm0;
+
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_F32));
+
+		xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
+
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSS_XMM_XMM, xmm0, tmp->tmp_low));
+
 		break;
 	}
 	case J_DOUBLE: {
-		struct var_info *xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
+		struct var_info *xmm0;
+
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_F64));
+
+		xmm0 = get_fixed_var(s->b_parent, MACH_REG_XMM0);
+
 		select_insn(s, tree, reg_reg_insn(INSN_MOVSD_XMM_XMM, xmm0, tmp->tmp_low));
+
 		break;
 	}
 	case J_LONG: {
-		struct var_info *eax = get_fixed_var(s->b_parent, MACH_REG_xAX);
-		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, tmp->tmp_low));
+		struct var_info *rax;
+
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
+
+		rax = get_fixed_var(s->b_parent, MACH_REG_RAX);
+
+		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, tmp->tmp_low));
+
 		break;
 	}
 	case J_INT: {
 		struct var_info *eax = get_fixed_var(s->b_parent, MACH_REG_xAX);
 
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I32));
+
 		if (vm_method_is_native(method)) {
 			/*
 			 * Native methods (especially JNI methods) might
@@ -2319,8 +2419,14 @@ static void save_invoke_result(struct basic_block *s,
 		break;
 	}
 	case J_REFERENCE: {
-		struct var_info *eax = get_fixed_var(s->b_parent, MACH_REG_xAX);
-		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, eax, tmp->tmp_low));
+		struct var_info *rax;
+
+		select_insn(s, tree, insn(INSN_RESTORE_CALLER_REGS_I64));
+
+		rax = get_fixed_var(s->b_parent, MACH_REG_RAX);
+
+		select_insn(s, tree, reg_reg_insn(INSN_MOV_REG_REG, rax, tmp->tmp_low));
+
 		break;
 	}
 	default:
@@ -2576,12 +2682,16 @@ static void select_eh_prologue(struct basic_block *bb)
 
 	/* Save exception from current_exec_env.exception to exception
 	 * stack slot */
+	eh_add_insn(bb, insn(INSN_SAVE_CALLER_REGS));
 	eh_add_insn(bb, rel_insn(INSN_CALL_REL, (unsigned long)vm_get_exec_env));
+	eh_add_insn(bb, insn(INSN_RESTORE_CALLER_REGS_I64));
 	eh_add_insn(bb, membase_reg_insn(INSN_MOV_MEMBASE_REG, reg_eax, offsetof(struct vm_exec_env, exception), reg_eax));
 	eh_add_insn(bb, reg_memlocal_insn(INSN_MOV_REG_MEMLOCAL, reg_eax, bb->b_parent->exception_spill_slot));
 
 	/* Clear exception. TODO: inline this. */
+	eh_add_insn(bb, insn(INSN_SAVE_CALLER_REGS));
 	eh_add_insn(bb, rel_insn(INSN_CALL_REL, (unsigned long)clear_exception));
+	eh_add_insn(bb, insn(INSN_RESTORE_CALLER_REGS));
 }
 
 static void insn_select(struct basic_block *bb)
diff --git a/arch/x86/instruction.c b/arch/x86/instruction.c
index 08f97c22..871ab2a2 100644
--- a/arch/x86/instruction.c
+++ b/arch/x86/instruction.c
@@ -906,6 +906,12 @@ static unsigned long insn_flags[] = {
 	[INSN_XOR_MEMBASE_REG]			= USE_SRC | USE_DST | DEF_DST,
 	[INSN_XOR_REG_REG]			= USE_SRC | USE_DST | DEF_DST,
 	[INSN_XORPS_XMM_XMM]			= USE_SRC | USE_DST | DEF_DST,
+	[INSN_SAVE_CALLER_REGS]			= USE_NONE | DEF_NONE,
+	[INSN_RESTORE_CALLER_REGS]		= USE_NONE | DEF_NONE,
+	[INSN_RESTORE_CALLER_REGS_I32]		= USE_NONE | DEF_NONE,
+	[INSN_RESTORE_CALLER_REGS_I64]		= USE_NONE | DEF_NONE,
+	[INSN_RESTORE_CALLER_REGS_F32]		= USE_NONE | DEF_NONE,
+	[INSN_RESTORE_CALLER_REGS_F64]		= USE_NONE | DEF_NONE,
 };
 
 void insn_sanity_check(void)
diff --git a/arch/x86/lir-printer.c b/arch/x86/lir-printer.c
index 2684e8db..f7420ce4 100644
--- a/arch/x86/lir-printer.c
+++ b/arch/x86/lir-printer.c
@@ -981,6 +981,36 @@ static int print_xor_64_xmm_reg_reg(struct string *str, struct insn *insn)
 	return print_reg_reg(str, insn);
 }
 
+static int print_save_caller_regs(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
+static int print_restore_caller_regs(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
+static int print_restore_caller_regs_i32(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
+static int print_restore_caller_regs_i64(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
+static int print_restore_caller_regs_f32(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
+static int print_restore_caller_regs_f64(struct string *str, struct insn *insn)
+{
+	return print_func_name(str);
+}
+
 typedef int (*print_insn_fn) (struct string *str, struct insn *insn);
 
 static print_insn_fn insn_printers[] = {
@@ -1106,6 +1136,12 @@ static print_insn_fn insn_printers[] = {
 	[INSN_XORPS_XMM_XMM] = print_xor_xmm_reg_reg,
 	[INSN_XOR_MEMBASE_REG] = print_xor_membase_reg,
 	[INSN_XOR_REG_REG] = print_xor_reg_reg,
+	[INSN_SAVE_CALLER_REGS] = print_save_caller_regs,
+	[INSN_RESTORE_CALLER_REGS] = print_restore_caller_regs,
+	[INSN_RESTORE_CALLER_REGS_I32] = print_restore_caller_regs_i32,
+	[INSN_RESTORE_CALLER_REGS_I64] = print_restore_caller_regs_i64,
+	[INSN_RESTORE_CALLER_REGS_F32] = print_restore_caller_regs_f32,
+	[INSN_RESTORE_CALLER_REGS_F64] = print_restore_caller_regs_f64,
 };
 
 int lir_print(struct insn *insn, struct string *str)
diff --git a/include/jit/compilation-unit.h b/include/jit/compilation-unit.h
index 39e8e0de..2732b4a8 100644
--- a/include/jit/compilation-unit.h
+++ b/include/jit/compilation-unit.h
@@ -71,6 +71,10 @@ struct compilation_unit {
 	/* It's needed to spill exception object reference at eh entry */
 	struct stack_slot *exception_spill_slot;
 
+	/* Array of spill slots used for caller-save register saving and
+	   restoring.  */
+	struct stack_slot *clobber_slots[NR_FIXED_REGISTERS];
+
 	/*
 	 * Pointers inside exit block and unwind block after monitor
 	 * unlocking code. The code is only present if method is
@@ -205,6 +209,7 @@ struct var_info *ssa_get_fixed_var(struct compilation_unit *, enum machine_reg);
 struct basic_block *find_bb(struct compilation_unit *, unsigned long);
 void compute_insn_positions(struct compilation_unit *);
 struct stack_slot *get_scratch_slot(struct compilation_unit *);
+struct stack_slot *get_clobber_slot(struct compilation_unit *cu, enum machine_reg reg);
 
 #define for_each_variable(var, var_list) for (var = var_list; var != NULL; var = var->next)
 
diff --git a/include/jit/compiler.h b/include/jit/compiler.h
index 358ebace..4ec15600 100644
--- a/include/jit/compiler.h
+++ b/include/jit/compiler.h
@@ -74,6 +74,7 @@ int dce(struct compilation_unit *cu);
 void imm_copy_propagation(struct compilation_unit *cu);
 void abc_removal(struct compilation_unit *cu);
 int allocate_registers(struct compilation_unit *cu);
+int mark_clobbers(struct compilation_unit *cu);
 int insert_spill_reload_insns(struct compilation_unit *cu);
 int emit_machine_code(struct compilation_unit *);
 void *jit_magic_trampoline(struct compilation_unit *);
diff --git a/include/jit/stack-slot.h b/include/jit/stack-slot.h
index cd865c9a..d08045dd 100644
--- a/include/jit/stack-slot.h
+++ b/include/jit/stack-slot.h
@@ -4,6 +4,7 @@
 #include "vm/types.h"
 
 struct stack_frame;
+enum machine_reg;
 
 struct stack_slot {
 	struct stack_frame *parent;
diff --git a/jit/clobber.c b/jit/clobber.c
new file mode 100644
index 00000000..e3a7c4b4
--- /dev/null
+++ b/jit/clobber.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2012  Pekka Enberg
+ *
+ * This file is released under the GPL version 2 with the following
+ * clarification and special exception:
+ *
+ *     Linking this library statically or dynamically with other modules is
+ *     making a combined work based on this library. Thus, the terms and
+ *     conditions of the GNU General Public License cover the whole
+ *     combination.
+ *
+ *     As a special exception, the copyright holders of this library give you
+ *     permission to link this library with independent modules to produce an
+ *     executable, regardless of the license terms of these independent
+ *     modules, and to copy and distribute the resulting executable under terms
+ *     of your choice, provided that you also meet, for each linked independent
+ *     module, the terms and conditions of the license of that module. An
+ *     independent module is a module which is not derived from or based on
+ *     this library. If you modify this library, you may extend this exception
+ *     to your version of the library, but you are not obligated to do so. If
+ *     you do not wish to do so, delete this exception statement from your
+ *     version.
+ *
+ * Please refer to the file LICENSE for details.
+ *
+ */
+
+#include "jit/compiler.h"
+
+#include "jit/bc-offset-mapping.h"
+#include "jit/compilation-unit.h"
+#include "jit/basic-block.h"
+#include "jit/instruction.h"
+
+#include <assert.h>
+
+static void
+insert_reload_after_insn(struct compilation_unit *cu, struct insn *insn, enum vm_type return_type)
+{
+	int i;
+
+	for (i = 0; i < NR_CALLER_SAVE_REGS; i++) {
+		enum machine_reg reg = caller_save_regs[i];
+		unsigned long bc_offset;
+		struct insn *insn_reload;
+		struct stack_slot *slot;
+		struct var_info *var;
+
+		if (is_return_reg(reg, return_type))
+			continue;
+
+		/* TODO: only spill/reload live registers */
+
+		slot = get_clobber_slot(cu, reg);
+
+		var = get_fixed_var(cu, reg);
+
+		insn_reload = reload_insn(slot, var);
+
+		bc_offset = insn_get_bc_offset(insn);
+
+		insn_set_bc_offset(insn_reload, bc_offset);
+
+		list_add(&insn_reload->insn_list_node, &insn->insn_list_node);
+	}
+}
+
+static void insert_spill_before_insn(struct compilation_unit *cu, struct insn *insn)
+{
+	int i;
+
+	for (i = 0; i < NR_CALLER_SAVE_REGS; i++) {
+		enum machine_reg reg = caller_save_regs[i];
+		unsigned long bc_offset;
+		struct insn *insn_spill;
+		struct stack_slot *slot;
+		struct var_info *var;
+
+		/* TODO: only spill/reload live registers */
+
+		slot = get_clobber_slot(cu, reg);
+
+		var = get_fixed_var(cu, reg);
+
+		insn_spill = spill_insn(var, slot);
+
+		bc_offset = insn_get_bc_offset(insn);
+
+		insn_set_bc_offset(insn_spill, bc_offset);
+
+		list_add_tail(&insn_spill->insn_list_node, &insn->insn_list_node);
+	}
+}
+
+int mark_clobbers(struct compilation_unit *cu)
+{
+	struct basic_block *bb;
+	struct insn *insn;
+
+	for_each_basic_block(bb, &cu->bb_list) {
+		list_for_each_entry(insn, &bb->insn_list, insn_list_node) {
+			switch (insn->type) {
+			case INSN_SAVE_CALLER_REGS:
+				insert_spill_before_insn(cu, insn);
+				break;
+			case INSN_RESTORE_CALLER_REGS_I32:
+				insert_reload_after_insn(cu, insn, J_INT);
+				break;
+			case INSN_RESTORE_CALLER_REGS_I64:
+				insert_reload_after_insn(cu, insn, J_LONG);
+				break;
+			case INSN_RESTORE_CALLER_REGS_F32:
+				insert_reload_after_insn(cu, insn, J_FLOAT);
+				break;
+			case INSN_RESTORE_CALLER_REGS_F64:
+				insert_reload_after_insn(cu, insn, J_DOUBLE);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
diff --git a/jit/compilation-unit.c b/jit/compilation-unit.c
index 9f4e0d43..0b2698af 100644
--- a/jit/compilation-unit.c
+++ b/jit/compilation-unit.c
@@ -398,3 +398,15 @@ struct stack_slot *get_scratch_slot(struct compilation_unit *cu)
 
 	return cu->scratch_slot;
 }
+
+struct stack_slot *get_clobber_slot(struct compilation_unit *cu, enum machine_reg reg)
+{
+	struct stack_slot *slot = cu->clobber_slots[reg];
+
+	if (slot)
+		return slot;
+
+	slot = cu->clobber_slots[reg] = get_spill_slot(cu->stack_frame, J_NATIVE_PTR);
+
+	return slot;
+}
diff --git a/jit/compiler.c b/jit/compiler.c
index 22e4c476..37e861c7 100644
--- a/jit/compiler.c
+++ b/jit/compiler.c
@@ -145,6 +145,10 @@ int compile(struct compilation_unit *cu)
 	if (err)
 		goto out;
 
+	err = mark_clobbers(cu);
+	if (err)
+		goto out;
+
 	err = insert_spill_reload_insns(cu);
 	if (err)
 		goto out;
diff --git a/jit/stack-slot.c b/jit/stack-slot.c
index 882cb20d..366e80a0 100644
--- a/jit/stack-slot.c
+++ b/jit/stack-slot.c
@@ -23,8 +23,12 @@
  *
  * Please refer to the file LICENSE for details.
  */
+
 #include "jit/stack-slot.h"
+
+#include "arch/registers.h"
 #include "vm/stdlib.h"
+
 #include <stdlib.h>
 
 struct stack_frame *alloc_stack_frame(unsigned long nr_args,