x86/ibt: Implement IBT+x86/ibt

As originally conceived, use UD1 based poison to seal IBT. This is also fatal
on !IBT hardware.

This requires all direct (tail) calls avoid ever touching ENDBR. To that
purpose rewrite them using the .call_sites and .tail_call_sites sections from
objtool --direct-call.

Since this is a wee bit tricky, stick this in a 'def_bool y' config option.

This again stacks 3 layers of relocation, just like the earlier callthunk patch.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>

9 files changed, 126 insertions, 13 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3762f41bb09297..31b6122b1baaa0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -1868,6 +1868,10 @@ config X86_KERNEL_IBT
 	  does significantly reduce the number of ENDBR instructions in the
 	  kernel image.
 
+config X86_KERNEL_IBT_PLUS
+	depends on X86_KERNEL_IBT
+	def_bool y
+
 config X86_INTEL_MEMORY_PROTECTION_KEYS
 	prompt "Memory Protection Keys"
 	def_bool y
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index f7f9e396236d73..e8970364e72981 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -98,6 +98,7 @@ extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern int alternatives_patched;
 
 extern void alternative_instructions(void);
+extern void apply_direct_call_offset(s32 *start, s32 *end);
 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
 extern void apply_retpolines(s32 *start, s32 *end);
 extern void apply_returns(s32 *start, s32 *end);
diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h
index 4a539e2b8bd7a4..82f82b48961d74 100644
--- a/arch/x86/include/asm/cfi.h
+++ b/arch/x86/include/asm/cfi.h
@@ -30,12 +30,12 @@
  * IBT:
  *
  * foo:
- *   endbr64
+ *   endbr64 / osp nop3 / ud1 0x0(%eax), %eax
  *   ... code here ...
  *   ret
  *
  * direct caller:
- *   call foo / call foo+4
+ *   call foo / call foo+4	# must be +4 when IBT+
  *
  * indirect caller:
  *   lea foo(%rip), %r11
@@ -49,12 +49,12 @@
  *   movl $0x12345678, %eax
  *				# 11 nops when CONFIG_CALL_PADDING
  * foo:
- *   endbr64			# when IBT
+ *   endbr64 / osp nop3 / ud1	# when IBT
  *   ... code here ...
  *   ret
  *
  * direct call:
- *   call foo			# / call foo+4 when IBT
+ *   call foo / call foo+4	# +4 possible with IBT, mandatory with IBT+
  *
  * indirect call:
  *   lea foo(%rip), %r11
@@ -71,16 +71,16 @@
  * __cfi_foo:
  *   endbr64
  *   subl 0x12345678, %r10d
- *   jz   foo
+ *   jz   foo+4
  *   ud2
  *   nop
  * foo:
- *   osp nop3			# was endbr64
+ *   osp nop3 / ud1 0x0(%eax), %eax	# was endbr64
  *   ... code here ...
  *   ret
  *
  * direct caller:
- *   call foo / call foo+4
+ *   call foo / call foo+4	# must be +4 when IBT+
  *
  * indirect caller:
  *   lea foo(%rip), %r11
diff --git a/arch/x86/include/asm/ibt.h b/arch/x86/include/asm/ibt.h
index f0ca5c0b1a6b28..27e493465c60a8 100644
--- a/arch/x86/include/asm/ibt.h
+++ b/arch/x86/include/asm/ibt.h
@@ -58,11 +58,20 @@ static __always_inline __attribute_const__ u32 gen_endbr(void)
 
 static __always_inline __attribute_const__ u32 gen_endbr_poison(void)
 {
+#ifdef CONFIG_X86_KERNEL_IBT_PLUS
+	/*
+	 * When we rewrite direct calls to +4, the endbr at +0 becomes unused,
+	 * poisong it with a UD1 to trip !IBT hardware and to ensure these
+	 * bytes are really unused.
+	 */
+	return 0x0040b90f; /* ud1 0x0(%eax), %eax */
+#else
 	/*
 	 * 4 byte NOP that isn't NOP4 (in fact it is OSP NOP3), such that it
 	 * will be unique to (former) ENDBR sites.
 	 */
 	return 0x001f0f66; /* osp nopl (%rax) */
+#endif
 }
 
 static inline bool __is_endbr(u32 val)
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 912014dd514a86..0268b68d6e582e 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -157,6 +157,8 @@ static void __init_or_module add_nop(u8 *instr, unsigned int len)
 		*instr = INT3_INSN_OPCODE;
 }
 
+extern s32 __call_sites[], __call_sites_end[];
+extern s32 __tail_call_sites[], __tail_call_sites_end[];
 extern s32 __retpoline_sites[], __retpoline_sites_end[];
 extern s32 __return_sites[], __return_sites_end[];
 extern s32 __cfi_sites[], __cfi_sites_end[];
@@ -835,6 +837,64 @@ void __init_or_module noinline apply_returns(s32 *start, s32 *end) { }
 
 #endif /* CONFIG_RETPOLINE && CONFIG_OBJTOOL */
 
+#ifdef CONFIG_X86_KERNEL_IBT_PLUS
+__init_or_module void apply_direct_call_offset(s32 *start, s32 *end)
+{
+	s32 *s;
+
+	/*
+	 * incompatible with call depth tracking
+	 */
+	if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
+		return;
+
+	for (s = start; s < end; s++) {
+		void *dest, *addr = (void *)s + *s;
+		struct insn insn;
+		int ret;
+
+		ret = insn_decode_kernel(&insn, addr);
+		if (WARN_ON_ONCE(ret < 0))
+			continue;
+
+		dest = addr + insn.length + insn.immediate.value;
+		if (!is_endbr(dest))
+			continue;
+
+		switch (insn.opcode.bytes[0]) {
+		case CALL_INSN_OPCODE:
+		case JMP32_INSN_OPCODE:
+			apply_reloc(4, addr+1, 4);
+			continue;
+
+		case JMP8_INSN_OPCODE:
+		case 0x70 ... 0x7f: /* Jcc.d8 */
+			apply_reloc(1, addr+1, 4);
+			continue;
+
+		case 0x0f:
+			switch (insn.opcode.bytes[1]) {
+			case 0x80 ... 0x8f:
+				apply_reloc(4, addr+2, 4);
+				continue;
+
+			default:
+				break;
+			}
+			break;
+
+		default:
+			break;
+		}
+
+		printk("at: %pS, instruction: %*ph\n", addr, insn.length, addr);
+		BUG();
+	}
+}
+#else
+__init_or_module void apply_direct_call_offset(s32 *start, s32 *end) { }
+#endif
+
 #ifdef CONFIG_X86_KERNEL_IBT
 
 __noendbr bool is_endbr(u32 *val)
@@ -1013,6 +1073,7 @@ asm(	".pushsection .rodata			\n"
 	"fineibt_preamble_start:		\n"
 	"	endbr64				\n"
 	"	subl	$0x12345678, %r10d	\n"
+	"fineibt_preamble_jcc:			\n"
 	"	je	fineibt_preamble_end	\n"
 	"	ud2				\n"
 	"	nop				\n"
@@ -1021,10 +1082,13 @@ asm(	".pushsection .rodata			\n"
 );
 
 extern u8 fineibt_preamble_start[];
+extern u8 fineibt_preamble_jcc[];
 extern u8 fineibt_preamble_end[];
 
-#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
-#define fineibt_preamble_hash 7
+#define fineibt_preamble_size	(fineibt_preamble_end - fineibt_preamble_start)
+#define fineibt_preamble_offset	(fineibt_preamble_jcc - fineibt_preamble_start)
+#define fineibt_preamble_hash	(fineibt_preamble_offset - 4)
+#define fineibt_preamble_jccd8	(fineibt_preamble_offset + 1)
 
 asm(	".pushsection .rodata			\n"
 	"fineibt_caller_start:			\n"
@@ -1158,6 +1222,8 @@ static int cfi_rewrite_preamble(s32 *start, s32 *end)
 		text_poke_early(addr, fineibt_preamble_start, fineibt_preamble_size);
 		WARN_ON(*(u32 *)(addr + fineibt_preamble_hash) != 0x12345678);
 		text_poke_early(addr + fineibt_preamble_hash, &hash, 4);
+
+		*(u8 *)(addr + fineibt_preamble_jccd8) += 4;
 	}
 
 	return 0;
@@ -1647,6 +1713,12 @@ void __init alternative_instructions(void)
 	 */
 	paravirt_set_cap();
 
+	/*
+	 * Adjust all (tail) calls to func()+4 to avoid ENDBR.
+	 */
+	apply_direct_call_offset(__call_sites, __call_sites_end);
+	apply_direct_call_offset(__tail_call_sites, __tail_call_sites_end);
+
 	__apply_fineibt(__retpoline_sites, __retpoline_sites_end,
 			__cfi_sites, __cfi_sites_end, true);
 
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 644396e56da52b..d87edec9a7f22d 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -278,7 +278,7 @@ int module_finalize(const Elf_Ehdr *hdr,
 	const Elf_Shdr *s, *alt = NULL, *locks = NULL,
 		*orc = NULL, *orc_ip = NULL,
 		*retpolines = NULL, *returns = NULL, *ibt_endbr = NULL,
-		*calls = NULL, *cfi = NULL;
+		*calls = NULL, *tails = NULL, *cfi = NULL;
 	char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
 	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -296,6 +296,8 @@ int module_finalize(const Elf_Ehdr *hdr,
 			returns = s;
 		if (!strcmp(".call_sites", secstrings + s->sh_name))
 			calls = s;
+		if (!strcmp(".tail_call_sites", secstrings + s->sh_name))
+			tails = s;
 		if (!strcmp(".cfi_sites", secstrings + s->sh_name))
 			cfi = s;
 		if (!strcmp(".ibt_endbr_seal", secstrings + s->sh_name))
@@ -335,6 +337,11 @@ int module_finalize(const Elf_Ehdr *hdr,
 		}
 
 		callthunks_patch_module_calls(&cs, me);
+		apply_direct_call_offset(cs.call_start, cs.call_end);
+	}
+	if (tails) {
+		void *tseg = (void *)tails->sh_addr;
+		apply_direct_call_offset(tseg, tseg + tails->sh_size);
 	}
 	if (alt) {
 		/* patch .altinstructions */
diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
index 77a9316da43573..422e77ffc23714 100644
--- a/arch/x86/kernel/static_call.c
+++ b/arch/x86/kernel/static_call.c
@@ -50,6 +50,23 @@ asm (".global __static_call_return\n\t"
      "ret; int3\n\t"
      ".size __static_call_return, . - __static_call_return \n\t");
 
+static void *translate_call_dest(void *dest, bool call)
+{
+	if (cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) {
+		if (!call)
+			return dest;
+
+		return callthunks_translate_call_dest(dest);
+	}
+
+	if (IS_ENABLED(CONFIG_X86_KERNEL_IBT_PLUS)) {
+		if (is_endbr(dest))
+			dest += 4;
+	}
+
+	return dest;
+}
+
 static void __ref __static_call_transform(void *insn, enum insn_type type,
 					  void *func, bool modinit)
 {
@@ -63,7 +80,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type,
 
 	switch (type) {
 	case CALL:
-		func = callthunks_translate_call_dest(func);
+		func = translate_call_dest(func, true);
 		code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
 		if (func == &__static_call_return0) {
 			emulate = code;
@@ -77,6 +94,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type,
 		break;
 
 	case JMP:
+		func = translate_call_dest(func, false);
 		code = text_gen_insn(JMP32_INSN_OPCODE, insn, func);
 		break;
 
@@ -92,7 +110,8 @@ static void __ref __static_call_transform(void *insn, enum insn_type type,
 			func = __static_call_return;
 			if (cpu_feature_enabled(X86_FEATURE_RETHUNK))
 				func = x86_return_thunk;
-		}
+
+		} else func = translate_call_dest(func, false);
 
 		buf[0] = 0x0f;
 		__text_gen_insn(buf+1, op, insn+1, func, 5);
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 13365328f2d39f..ad960dbfd221ca 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -256,6 +256,7 @@ objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK)		+= --hacks=jump_label
 objtool-args-$(CONFIG_HAVE_NOINSTR_HACK)		+= --hacks=noinstr
 objtool-args-$(CONFIG_CALL_DEPTH_TRACKING)		+= --direct-call
 objtool-args-$(CONFIG_X86_KERNEL_IBT)			+= --ibt
+objtool-args-$(CONFIG_X86_KERNEL_IBT_PLUS)		+= --direct-call
 objtool-args-$(CONFIG_FINEIBT)				+= --cfi
 objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL)	+= --mcount
 ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index dc7977b66449a4..57b45fb708b35c 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -1450,7 +1450,7 @@ static void annotate_call_site(struct objtool_file *file,
 		return;
 	}
 
-	if (!insn->sec->init && !insn->_call_dest->embedded_insn) {
+	if (!insn->_call_dest->embedded_insn) {
 		if (insn->type == INSN_CALL)
 			list_add_tail(&insn->call_node, &file->call_list);
 		else