x86: Permit absolute per-CPU references in asm codex86-pie

We have code in asm that is either part of a template that gets copied
around, or is in an alternative block that gets patched into a different
location in the binary.

This means we cannot use RIP-relative addressing here unless we fix up
the relative offset in the instruction after copying the code, which is
kind of tricky.

Instead, let's permit 32-bit sign extended per-CPU references, by
tracking their locations and targets, and doing a pass at boot to insert
the correct immediates.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>

3 files changed, 44 insertions, 6 deletions

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index bc2f4892a758b5..1f12247c60188d 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -78,28 +78,37 @@
 #include <asm/asm-offsets.h>
 
 #define CREDIT_CALL_DEPTH					\
-	movq	$-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
+	ABS_PERCPU_REF 0x5, pcpu_hot + X86_call_depth;		\
+	movq	$-1, %gs:0x0;
 
 #define ASM_CREDIT_CALL_DEPTH					\
-	movq	$-1, PER_CPU_VAR(pcpu_hot + X86_call_depth);
+	ABS_PERCPU_REF 0x5, pcpu_hot + X86_call_depth;		\
+	movq	$-1, %gs:0x0;
 
 #define RESET_CALL_DEPTH					\
 	mov	$0x80, %rax;					\
 	shl	$56, %rax;					\
-	movq	%rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);
+	ABS_PERCPU_REF 0x5, pcpu_hot + X86_call_depth;		\
+	movq	%rax, %gs:0x0;
 
 #define RESET_CALL_DEPTH_FROM_CALL				\
 	mov	$0xfc, %rax;					\
 	shl	$56, %rax;					\
-	movq	%rax, PER_CPU_VAR(pcpu_hot + X86_call_depth);	\
+	ABS_PERCPU_REF 0x5, pcpu_hot + X86_call_depth;		\
+	movq	%rax, %gs:0x0;					\
 	CALL_THUNKS_DEBUG_INC_CALLS
 
 #define INCREMENT_CALL_DEPTH					\
-	sarq	$5, %gs:pcpu_hot + X86_call_depth;		\
+	.pushsection	".reltab.32s","a",@progbits;		\
+	.long	.Lpcp + 0x5 - .;				\
+	.long	pcpu_hot + X86_call_depth - .;			\
+	.popsection;						\
+.Lpcp:	sarq	$5, %gs:0x0;					\
 	CALL_THUNKS_DEBUG_INC_CALLS
 
 #define ASM_INCREMENT_CALL_DEPTH				\
-	sarq	$5, PER_CPU_VAR(pcpu_hot + X86_call_depth);	\
+	ABS_PERCPU_REF 0x5, pcpu_hot + X86_call_depth;		\
+	sarq	$5, %gs:0x0;					\
 	CALL_THUNKS_DEBUG_INC_CALLS
 
 #else
@@ -188,6 +197,18 @@
 
 #ifdef __ASSEMBLY__
 
+.macro ABS_PERCPU_REF offset, sym
+.Lplace_\@:
+#if defined(CONFIG_X86_64_PIE) && !defined(MODULE)
+	.pushsection	".reltab.32s","a",@progbits
+	.long	.Lplace_\@ + (\offset) - .
+	.long	\sym - .
+	.popsection
+#else
+	.reloc	.Lplace_\@ + (\offset), R_X86_64_32S, \sym
+#endif
+.endm
+
 /*
  * This should be used immediately before an indirect jump/call. It tells
  * objtool the subsequent indirect jump/call is vouched safe for retpoline
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 6db30f4e653215..c1de1c08f1c452 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -633,4 +633,15 @@ void __init startup_64_setup_env(u64 va_shift)
 			place += 63;
 		}
 	}
+
+	if (IS_ENABLED(CONFIG_X86_64_PIE)) {
+		extern const s32 __reltab_start[], __reltab_end[];
+
+		for (const s32 *r = __reltab_start; r < __reltab_end; r++) {
+			u32 *place = offset_to_ptr(r++);
+			void *sym = offset_to_ptr(r);
+
+			*place = (u64)sym - va_offset + va_shift;
+		}
+	}
 }
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index d0e688cf24c5a1..1a3bb6626138f2 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -247,6 +247,12 @@ SECTIONS
 		__relr_end = .;
 	}
 
+	.reltab.32s : {
+		__reltab_start = .;
+		*(.reltab.32s)
+		__reltab_end = .;
+	}
+
 	/*
 	 * Section for code used exclusively before alternatives are run. All
 	 * references to such code must be patched out by alternatives, normally