diff options
Diffstat (limited to 'queue-6.6/net-fix-sk_memory_allocated_-add-sub-vs-softirqs.patch')
| -rw-r--r-- | queue-6.6/net-fix-sk_memory_allocated_-add-sub-vs-softirqs.patch | 102 |
1 files changed, 102 insertions, 0 deletions
diff --git a/queue-6.6/net-fix-sk_memory_allocated_-add-sub-vs-softirqs.patch b/queue-6.6/net-fix-sk_memory_allocated_-add-sub-vs-softirqs.patch new file mode 100644 index 0000000000..1b4b071090 --- /dev/null +++ b/queue-6.6/net-fix-sk_memory_allocated_-add-sub-vs-softirqs.patch @@ -0,0 +1,102 @@ +From a8ca60debb0f2ef1726c2a61120ac7f458e75221 Mon Sep 17 00:00:00 2001 +From: Sasha Levin <sashal@kernel.org> +Date: Sun, 21 Apr 2024 17:52:48 +0000 +Subject: net: fix sk_memory_allocated_{add|sub} vs softirqs + +From: Eric Dumazet <edumazet@google.com> + +[ Upstream commit 3584718cf2ec7e79b6814f2596dcf398c5fb2eca ] + +Jonathan Heathcote reported a regression caused by blamed commit +on aarch64 architecture. + +x86 happens to have irq-safe __this_cpu_add_return() +and __this_cpu_sub(), but this is not generic. + +I think my confusion came from "struct sock" argument, +because these helpers are called with a locked socket. +But the memory accounting is per-proto (and per-cpu after +the blamed commit). We might cleanup these helpers later +to directly accept a "struct proto *proto" argument. + +Switch to this_cpu_add_return() and this_cpu_xchg() +operations, and get rid of preempt_disable()/preempt_enable() pairs. + +Fast path becomes a bit faster as a result :) + +Many thanks to Jonathan Heathcote for his awesome report and +investigations. + +Fixes: 3cd3399dd7a8 ("net: implement per-cpu reserves for memory_allocated") +Reported-by: Jonathan Heathcote <jonathan.heathcote@bbc.co.uk> +Closes: https://lore.kernel.org/netdev/VI1PR01MB42407D7947B2EA448F1E04EFD10D2@VI1PR01MB4240.eurprd01.prod.exchangelabs.com/ +Signed-off-by: Eric Dumazet <edumazet@google.com> +Acked-by: Soheil Hassas Yeganeh <soheil@google.com> +Reviewed-by: Shakeel Butt <shakeel.butt@linux.dev> +Link: https://lore.kernel.org/r/20240421175248.1692552-1-edumazet@google.com +Signed-off-by: Jakub Kicinski <kuba@kernel.org> +Signed-off-by: Sasha Levin <sashal@kernel.org> +--- + include/net/sock.h | 38 ++++++++++++++++++++------------------ + 1 file changed, 20 insertions(+), 18 deletions(-) + +diff --git a/include/net/sock.h b/include/net/sock.h +index 4372a7225ae6a..53b81e0a89810 100644 +--- a/include/net/sock.h ++++ b/include/net/sock.h +@@ -1460,32 +1460,34 @@ sk_memory_allocated(const struct sock *sk) + #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) + extern int sysctl_mem_pcpu_rsv; + ++static inline void proto_memory_pcpu_drain(struct proto *proto) ++{ ++ int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0); ++ ++ if (val) ++ atomic_long_add(val, proto->memory_allocated); ++} ++ + static inline void +-sk_memory_allocated_add(struct sock *sk, int amt) ++sk_memory_allocated_add(const struct sock *sk, int val) + { +- int local_reserve; ++ struct proto *proto = sk->sk_prot; + +- preempt_disable(); +- local_reserve = __this_cpu_add_return(*sk->sk_prot->per_cpu_fw_alloc, amt); +- if (local_reserve >= READ_ONCE(sysctl_mem_pcpu_rsv)) { +- __this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve); +- atomic_long_add(local_reserve, sk->sk_prot->memory_allocated); +- } +- preempt_enable(); ++ val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val); ++ ++ if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv))) ++ proto_memory_pcpu_drain(proto); + } + + static inline void +-sk_memory_allocated_sub(struct sock *sk, int amt) ++sk_memory_allocated_sub(const struct sock *sk, int val) + { +- int local_reserve; ++ struct proto *proto = sk->sk_prot; + +- preempt_disable(); +- local_reserve = __this_cpu_sub_return(*sk->sk_prot->per_cpu_fw_alloc, amt); +- if (local_reserve <= -READ_ONCE(sysctl_mem_pcpu_rsv)) { +- __this_cpu_sub(*sk->sk_prot->per_cpu_fw_alloc, local_reserve); +- atomic_long_add(local_reserve, sk->sk_prot->memory_allocated); +- } +- preempt_enable(); ++ val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val); ++ ++ if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv))) ++ proto_memory_pcpu_drain(proto); + } + + #define SK_ALLOC_PERCPU_COUNTER_BATCH 16 +-- +2.43.0 + |