dst_cache.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * net/core/dst_cache.c - dst entry cache
  4 *
  5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/percpu.h>
 10#include <net/dst_cache.h>
 11#include <net/route.h>
 12#if IS_ENABLED(CONFIG_IPV6)
 13#include <net/ip6_fib.h>
 14#endif
 15#include <uapi/linux/in.h>
 16
 17struct dst_cache_pcpu {
 18	unsigned long refresh_ts;
 19	struct dst_entry *dst;
 20	local_lock_t bh_lock;
 21	u32 cookie;
 22	union {
 23		struct in_addr in_saddr;
 24		struct in6_addr in6_saddr;
 25	};
 26};
 27
 28static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
 29				      struct dst_entry *dst, u32 cookie)
 30{
 31	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
 32	dst_release(dst_cache->dst);
 33	if (dst)
 34		dst_hold(dst);
 35
 36	dst_cache->cookie = cookie;
 37	dst_cache->dst = dst;
 38}
 39
 40static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
 41					       struct dst_cache_pcpu *idst)
 42{
 43	struct dst_entry *dst;
 44
 45	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
 46	dst = idst->dst;
 47	if (!dst)
 48		goto fail;
 49
 50	/* the cache already hold a dst reference; it can't go away */
 51	dst_hold(dst);
 52
 53	if (unlikely(!time_after(idst->refresh_ts,
 54				 READ_ONCE(dst_cache->reset_ts)) ||
 55		     (READ_ONCE(dst->obsolete) && !dst->ops->check(dst, idst->cookie)))) {
 56		dst_cache_per_cpu_dst_set(idst, NULL, 0);
 57		dst_release(dst);
 58		goto fail;
 59	}
 60	return dst;
 61
 62fail:
 63	idst->refresh_ts = jiffies;
 64	return NULL;
 65}
 66
 67struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
 68{
 69	struct dst_entry *dst;
 70
 71	if (!dst_cache->cache)
 72		return NULL;
 73
 74	local_lock_nested_bh(&dst_cache->cache->bh_lock);
 75	dst = dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
 76	local_unlock_nested_bh(&dst_cache->cache->bh_lock);
 77	return dst;
 78}
 79EXPORT_SYMBOL_GPL(dst_cache_get);
 80
 81struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
 82{
 83	struct dst_cache_pcpu *idst;
 84	struct dst_entry *dst;
 85
 86	if (!dst_cache->cache)
 87		return NULL;
 88
 89	local_lock_nested_bh(&dst_cache->cache->bh_lock);
 90	idst = this_cpu_ptr(dst_cache->cache);
 91	dst = dst_cache_per_cpu_get(dst_cache, idst);
 92	if (!dst) {
 93		local_unlock_nested_bh(&dst_cache->cache->bh_lock);
 94		return NULL;
 95	}
 96
 97	*saddr = idst->in_saddr.s_addr;
 98	local_unlock_nested_bh(&dst_cache->cache->bh_lock);
 99	return dst_rtable(dst);
100}
101EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
102
103void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
104		       __be32 saddr)
105{
106	struct dst_cache_pcpu *idst;
107
108	if (!dst_cache->cache)
109		return;
110
111	local_lock_nested_bh(&dst_cache->cache->bh_lock);
112	idst = this_cpu_ptr(dst_cache->cache);
113	dst_cache_per_cpu_dst_set(idst, dst, 0);
114	idst->in_saddr.s_addr = saddr;
115	local_unlock_nested_bh(&dst_cache->cache->bh_lock);
116}
117EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
118
119#if IS_ENABLED(CONFIG_IPV6)
120void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
121		       const struct in6_addr *saddr)
122{
123	struct dst_cache_pcpu *idst;
124
125	if (!dst_cache->cache)
126		return;
127
128	local_lock_nested_bh(&dst_cache->cache->bh_lock);
129
130	idst = this_cpu_ptr(dst_cache->cache);
131	dst_cache_per_cpu_dst_set(idst, dst,
132				  rt6_get_cookie(dst_rt6_info(dst)));
133	idst->in6_saddr = *saddr;
134	local_unlock_nested_bh(&dst_cache->cache->bh_lock);
135}
136EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
137
138struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
139				    struct in6_addr *saddr)
140{
141	struct dst_cache_pcpu *idst;
142	struct dst_entry *dst;
143
144	if (!dst_cache->cache)
145		return NULL;
146
147	local_lock_nested_bh(&dst_cache->cache->bh_lock);
148
149	idst = this_cpu_ptr(dst_cache->cache);
150	dst = dst_cache_per_cpu_get(dst_cache, idst);
151	if (!dst) {
152		local_unlock_nested_bh(&dst_cache->cache->bh_lock);
153		return NULL;
154	}
155
156	*saddr = idst->in6_saddr;
157	local_unlock_nested_bh(&dst_cache->cache->bh_lock);
158	return dst;
159}
160EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
161#endif
162
163int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
164{
165	unsigned int i;
166
167	dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
168					    gfp | __GFP_ZERO);
169	if (!dst_cache->cache)
170		return -ENOMEM;
171	for_each_possible_cpu(i)
172		local_lock_init(&per_cpu_ptr(dst_cache->cache, i)->bh_lock);
173
174	dst_cache_reset(dst_cache);
175	return 0;
176}
177EXPORT_SYMBOL_GPL(dst_cache_init);
178
179void dst_cache_destroy(struct dst_cache *dst_cache)
180{
181	int i;
182
183	if (!dst_cache->cache)
184		return;
185
186	for_each_possible_cpu(i)
187		dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
188
189	free_percpu(dst_cache->cache);
190}
191EXPORT_SYMBOL_GPL(dst_cache_destroy);
192
193void dst_cache_reset_now(struct dst_cache *dst_cache)
194{
195	int i;
196
197	if (!dst_cache->cache)
198		return;
199
200	dst_cache_reset(dst_cache);
201	for_each_possible_cpu(i) {
202		struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
203		struct dst_entry *dst = idst->dst;
204
205		idst->cookie = 0;
206		idst->dst = NULL;
207		dst_release(dst);
208	}
209}
210EXPORT_SYMBOL_GPL(dst_cache_reset_now);