bpf/tests: add performance and correctness tests for LD_ABS_DWld_abs_dw

net.core.bpf_jit_enable = 0
[   29.246177] test_bpf: #305 perf: LD_ABS byte jited:0 52 PASS
[   29.248082] test_bpf: #306 perf: LD_ABS_DW byte jited:0 44 PASS
[   29.249256] test_bpf: #307 perf: LD_ABS word jited:0 62 PASS
[   29.250848] test_bpf: #308 perf: LD_ABS_DW word jited:0 35 PASS
[   29.251958] test_bpf: #309 perf: LD_IND byte jited:0 53 PASS
[   29.253224] test_bpf: #310 perf: LD_IND_DW byte jited:0 44 PASS
[   29.254398] test_bpf: Summary: 6 PASSED, 0 FAILED, [0/6 JIT'ed]
net.core.bpf_jit_enable = 1
[   29.259261] test_bpf: #305 perf: LD_ABS byte jited:1 18 PASS
[   29.260416] test_bpf: #306 perf: LD_ABS_DW byte jited:1 5 PASS
[   29.261216] test_bpf: #307 perf: LD_ABS word jited:1 24 PASS
[   29.262268] test_bpf: #308 perf: LD_ABS_DW word jited:1 5 PASS
[   29.263121] test_bpf: #309 perf: LD_IND byte jited:1 20 PASS
[   29.264064] test_bpf: #310 perf: LD_IND_DW byte jited:1 5 PASS

ld_abs_dw style of packet access is slightly faster in interpreter
and dramatically faster after JIT.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>

2 files changed, 167 insertions, 0 deletions

diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 8f22fbedc3a699..35df44119f2c7d 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -5460,6 +5460,131 @@ static struct bpf_test tests[] = {
 		{},
 		{ {0x1, 0x42 } },
 	},
+	{
+		"perf: LD_ABS byte",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_LD_ABS(BPF_B, 0),
+			BPF_LD_ABS(BPF_B, 1),
+			BPF_LD_ABS(BPF_B, 2),
+			BPF_LD_ABS(BPF_B, 3),
+			BPF_LD_ABS(BPF_B, 4),
+			BPF_LD_ABS(BPF_B, 5),
+			BPF_LD_ABS(BPF_B, 6),
+			BPF_LD_ABS(BPF_B, 7),
+			BPF_LD_ABS(BPF_B, 8),
+			BPF_LD_ABS(BPF_B, 9),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55 },
+		{ { 10, 55 }}
+	},
+	{
+		"perf: LD_ABS_DW byte",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_RAW_INSN(BPF_LD|BPF_ABS|BPF_DW, R1, R0, 0, 10),
+			BPF_LDX_MEM(BPF_B, R0, R1, 0),
+			BPF_LDX_MEM(BPF_B, R0, R1, 1),
+			BPF_LDX_MEM(BPF_B, R0, R1, 2),
+			BPF_LDX_MEM(BPF_B, R0, R1, 3),
+			BPF_LDX_MEM(BPF_B, R0, R1, 4),
+			BPF_LDX_MEM(BPF_B, R0, R1, 5),
+			BPF_LDX_MEM(BPF_B, R0, R1, 6),
+			BPF_LDX_MEM(BPF_B, R0, R1, 7),
+			BPF_LDX_MEM(BPF_B, R0, R1, 8),
+			BPF_LDX_MEM(BPF_B, R0, R1, 9),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55 },
+		{ { 10, 55 }}
+	},
+	{
+		"perf: LD_ABS word",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_LD_ABS(BPF_W, 0),
+			BPF_LD_ABS(BPF_W, 1),
+			BPF_LD_ABS(BPF_W, 2),
+			BPF_LD_ABS(BPF_W, 3),
+			BPF_LD_ABS(BPF_W, 4),
+			BPF_LD_ABS(BPF_W, 5),
+			BPF_LD_ABS(BPF_W, 6),
+			BPF_LD_ABS(BPF_W, 7),
+			BPF_LD_ABS(BPF_W, 8),
+			BPF_LD_ABS(BPF_B, 9),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55, 0, 0 },
+		{ { 12, 55 }}
+	},
+#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+	{
+		"perf: LD_ABS_DW word",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_RAW_INSN(BPF_LD|BPF_ABS|BPF_DW, R1, R0, 0, 12),
+			BPF_LDX_MEM(BPF_W, R0, R1, 0),
+			BPF_LDX_MEM(BPF_W, R0, R1, 1),
+			BPF_LDX_MEM(BPF_W, R0, R1, 2),
+			BPF_LDX_MEM(BPF_W, R0, R1, 3),
+			BPF_LDX_MEM(BPF_W, R0, R1, 4),
+			BPF_LDX_MEM(BPF_W, R0, R1, 5),
+			BPF_LDX_MEM(BPF_W, R0, R1, 6),
+			BPF_LDX_MEM(BPF_W, R0, R1, 7),
+			BPF_LDX_MEM(BPF_W, R0, R1, 8),
+			BPF_LDX_MEM(BPF_B, R0, R1, 9),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55, 0, 0 },
+		{ { 12, 55 }}
+	},
+#endif
+	{
+		"perf: LD_IND byte",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_MOV64_IMM(R7, 1),
+			BPF_LD_IND(BPF_B, R7, 0),
+			BPF_LD_IND(BPF_B, R7, 1),
+			BPF_LD_IND(BPF_B, R7, 2),
+			BPF_LD_IND(BPF_B, R7, 3),
+			BPF_LD_IND(BPF_B, R7, 4),
+			BPF_LD_IND(BPF_B, R7, 5),
+			BPF_LD_IND(BPF_B, R7, 6),
+			BPF_LD_IND(BPF_B, R7, 7),
+			BPF_LD_IND(BPF_B, R7, 8),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55 },
+		{ { 10, 55 }}
+	},
+	{
+		"perf: LD_IND_DW byte",
+		.u.insns_int = {
+			BPF_MOV64_REG(R6, R1),
+			BPF_MOV64_IMM(R2, 1),
+			BPF_RAW_INSN(BPF_LD|BPF_IND|BPF_DW, R7, R2, 0, 9),
+			BPF_LDX_MEM(BPF_B, R0, R7, 0),
+			BPF_LDX_MEM(BPF_B, R0, R7, 1),
+			BPF_LDX_MEM(BPF_B, R0, R7, 2),
+			BPF_LDX_MEM(BPF_B, R0, R7, 3),
+			BPF_LDX_MEM(BPF_B, R0, R7, 4),
+			BPF_LDX_MEM(BPF_B, R0, R7, 5),
+			BPF_LDX_MEM(BPF_B, R0, R7, 6),
+			BPF_LDX_MEM(BPF_B, R0, R7, 7),
+			BPF_LDX_MEM(BPF_B, R0, R7, 8),
+			BPF_EXIT_INSN(),
+		},
+		INTERNAL,
+		{ 10, 20, 30, 40, 50, 11, 22, 33, 44, 55 },
+		{ { 10, 55 }}
+	},
 };
 
 static struct net_device dev;
diff --git a/samples/bpf/test_verifier.c b/samples/bpf/test_verifier.c
index 4b51a9039c0d4a..2048d1e9f636b5 100644
--- a/samples/bpf/test_verifier.c
+++ b/samples/bpf/test_verifier.c
@@ -1180,6 +1180,48 @@ static struct bpf_test tests[] = {
 		.result_unpriv = REJECT,
 		.result = ACCEPT,
 	},
+	{
+		"ld_abs_dw: test 1",
+		.insns = {
+			BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
+			BPF_RAW_INSN(BPF_LD | BPF_ABS | BPF_DW, 1, 0, 14, 1),
+			BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, 0),
+			BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, 1),
+			BPF_MOV64_IMM(BPF_REG_0, 0),
+			BPF_EXIT_INSN(),
+		},
+		.errstr = "invalid access to packet, off=1",
+		.result = REJECT,
+	},
+	{
+		"ld_abs_dw: test 2",
+		.insns = {
+			BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
+			BPF_RAW_INSN(BPF_LD | BPF_ABS | BPF_DW, 1, 0, 1, 7),
+			BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 2),
+			BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 1),
+			BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
+			BPF_MOV64_IMM(BPF_REG_0, 0),
+			BPF_EXIT_INSN(),
+		},
+		.errstr = "invalid access to packet, off=0",
+		.result = REJECT,
+	},
+	{
+		"ld_abs_dw: test 3",
+		.insns = {
+			BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
+			BPF_MOV64_IMM(BPF_REG_2, 10),
+			BPF_RAW_INSN(BPF_LD | BPF_IND | BPF_DW, 1, 2, 1, 7),
+			BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 2),
+			BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, 1),
+			BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0),
+			BPF_MOV64_IMM(BPF_REG_0, 0),
+			BPF_EXIT_INSN(),
+		},
+		.errstr = "invalid access to packet, off=0",
+		.result = REJECT,
+	},
 };
 
 static int probe_filter_length(struct bpf_insn *fp)