Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions drivers/net/vrf.c
Original file line number Diff line number Diff line change
Expand Up @@ -654,18 +654,18 @@ static int vrf_finish_output6(struct net *net, struct sock *sk,
skb->protocol = htons(ETH_P_IPV6);
skb->dev = dev;

rcu_read_lock_bh();
rcu_read_lock();
nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr);
neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
if (unlikely(!neigh))
neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
if (!IS_ERR(neigh)) {
sock_confirm_neigh(skb, neigh);
ret = neigh_output(neigh, skb, false);
rcu_read_unlock_bh();
rcu_read_unlock();
return ret;
}
rcu_read_unlock_bh();
rcu_read_unlock();

IP6_INC_STATS(dev_net(dst->dev),
ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
Expand Down Expand Up @@ -879,7 +879,7 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s
}
}

rcu_read_lock_bh();
rcu_read_lock();

neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
if (!IS_ERR(neigh)) {
Expand All @@ -888,11 +888,11 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s
sock_confirm_neigh(skb, neigh);
/* if crossing protocols, can not use the cached header */
ret = neigh_output(neigh, skb, is_v6gw);
rcu_read_unlock_bh();
rcu_read_unlock();
return ret;
}

rcu_read_unlock_bh();
rcu_read_unlock();
vrf_tx_error(skb->dev, skb);
return -EINVAL;
}
Expand Down
10 changes: 0 additions & 10 deletions include/linux/rtnetlink.h
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,6 @@ static inline bool lockdep_rtnl_is_held(void)
#define rcu_dereference_rtnl(p) \
rcu_dereference_check(p, lockdep_rtnl_is_held())

/**
* rcu_dereference_bh_rtnl - rcu_dereference_bh with debug checking
* @p: The pointer to read, prior to dereference
*
* Do an rcu_dereference_bh(p), but check caller either holds rcu_read_lock_bh()
* or RTNL. Note : Please prefer rtnl_dereference() or rcu_dereference_bh()
*/
#define rcu_dereference_bh_rtnl(p) \
rcu_dereference_bh_check(p, lockdep_rtnl_is_held())

/**
* rtnl_dereference - fetch RCU pointer when updates are prevented by RTNL
* @p: The pointer to read, prior to dereferencing
Expand Down
16 changes: 5 additions & 11 deletions include/net/arp.h
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,11 @@ static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv4_neigh_lookup_noref(dev, key);
if (n && !refcount_inc_not_zero(&n->refcnt))
n = NULL;
rcu_read_unlock_bh();
rcu_read_unlock();

return n;
}
Expand All @@ -51,16 +51,10 @@ static inline void __ipv4_confirm_neigh(struct net_device *dev, u32 key)
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv4_neigh_lookup_noref(dev, key);
if (n) {
unsigned long now = jiffies;

/* avoid dirtying neighbour */
if (READ_ONCE(n->confirmed) != now)
WRITE_ONCE(n->confirmed, now);
}
rcu_read_unlock_bh();
neigh_confirm(n);
rcu_read_unlock();
}

void arp_init(void);
Expand Down
42 changes: 41 additions & 1 deletion include/net/dst.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@
struct sk_buff;

struct dst_entry {
struct net_device *dev;
RH_KABI_REPLACE(
struct net_device *dev,
union {
struct net_device *dev;
struct net_device __rcu *dev_rcu;
})
struct dst_ops *ops;
unsigned long _metrics;
unsigned long expires;
Expand Down Expand Up @@ -579,6 +584,41 @@ static inline void skb_dst_update_pmtu_no_confirm(struct sk_buff *skb, u32 mtu)
dst->ops->update_pmtu(dst, NULL, skb, mtu, false);
}

static inline struct net_device *dst_dev(const struct dst_entry *dst)
{
return READ_ONCE(dst->dev);
}

static inline struct net_device *dst_dev_rcu(const struct dst_entry *dst)
{
return rcu_dereference(dst->dev_rcu);
}

static inline struct net *dst_dev_net_rcu(const struct dst_entry *dst)
{
return dev_net_rcu(dst_dev_rcu(dst));
}

static inline struct net_device *skb_dst_dev(const struct sk_buff *skb)
{
return dst_dev(skb_dst(skb));
}

static inline struct net_device *skb_dst_dev_rcu(const struct sk_buff *skb)
{
return dst_dev_rcu(skb_dst(skb));
}

static inline struct net *skb_dst_dev_net(const struct sk_buff *skb)
{
return dev_net(skb_dst_dev(skb));
}

static inline struct net *skb_dst_dev_net_rcu(const struct sk_buff *skb)
{
return dev_net_rcu(skb_dst_dev_rcu(skb));
}

struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie);
void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
struct sk_buff *skb, u32 mtu, bool confirm_neigh);
Expand Down
37 changes: 30 additions & 7 deletions include/net/ip.h
Original file line number Diff line number Diff line change
Expand Up @@ -443,20 +443,43 @@ static inline bool ip_sk_ignore_df(const struct sock *sk)
static inline unsigned int ip_dst_mtu_maybe_forward(const struct dst_entry *dst,
bool forwarding)
{
struct net *net = dev_net(dst->dev);
unsigned int mtu;
const struct rtable *rt = container_of(dst, struct rtable, dst);
const struct net_device *dev;
unsigned int mtu, res;
struct net *net;

rcu_read_lock();

dev = dst_dev_rcu(dst);
net = dev_net_rcu(dev);
if (READ_ONCE(net->ipv4.sysctl_ip_fwd_use_pmtu) ||
ip_mtu_locked(dst) ||
!forwarding)
return dst_mtu(dst);
!forwarding) {
mtu = rt->rt_pmtu;
if (mtu && time_before(jiffies, rt->dst.expires))
goto out;
}

/* 'forwarding = true' case should always honour route mtu */
mtu = dst_metric_raw(dst, RTAX_MTU);
if (!mtu)
mtu = min(READ_ONCE(dst->dev->mtu), IP_MAX_MTU);
if (mtu)
goto out;

mtu = READ_ONCE(dev->mtu);

if (unlikely(ip_mtu_locked(dst))) {
if (rt->rt_uses_gateway && mtu > 576)
mtu = 576;
}

out:
mtu = min_t(unsigned int, mtu, IP_MAX_MTU);

res = mtu - lwtunnel_headroom(dst->lwtstate, mtu);

rcu_read_unlock();

return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
return res;
}

static inline unsigned int ip_skb_dst_mtu(struct sock *sk,
Expand Down
2 changes: 1 addition & 1 deletion include/net/ip6_route.h
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ static inline unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst)

mtu = IPV6_MIN_MTU;
rcu_read_lock();
idev = __in6_dev_get(dst->dev);
idev = __in6_dev_get(dst_dev_rcu(dst));
if (idev)
mtu = READ_ONCE(idev->cnf.mtu6);
rcu_read_unlock();
Expand Down
28 changes: 8 additions & 20 deletions include/net/ndisc.h
Original file line number Diff line number Diff line change
Expand Up @@ -395,11 +395,11 @@ static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, cons
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref(dev, pkey);
if (n && !refcount_inc_not_zero(&n->refcnt))
n = NULL;
rcu_read_unlock_bh();
rcu_read_unlock();

return n;
}
Expand All @@ -409,33 +409,21 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev,
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref(dev, pkey);
if (n) {
unsigned long now = jiffies;

/* avoid dirtying neighbour */
if (READ_ONCE(n->confirmed) != now)
WRITE_ONCE(n->confirmed, now);
}
rcu_read_unlock_bh();
neigh_confirm(n);
rcu_read_unlock();
}

static inline void __ipv6_confirm_neigh_stub(struct net_device *dev,
const void *pkey)
{
struct neighbour *n;

rcu_read_lock_bh();
rcu_read_lock();
n = __ipv6_neigh_lookup_noref_stub(dev, pkey);
if (n) {
unsigned long now = jiffies;

/* avoid dirtying neighbour */
if (READ_ONCE(n->confirmed) != now)
WRITE_ONCE(n->confirmed, now);
}
rcu_read_unlock_bh();
neigh_confirm(n);
rcu_read_unlock();
}

/* uses ipv6_stub and is meant for use outside of IPv6 core */
Expand Down
17 changes: 14 additions & 3 deletions include/net/neighbour.h
Original file line number Diff line number Diff line change
Expand Up @@ -308,14 +308,14 @@ static inline struct neighbour *___neigh_lookup_noref(
const void *pkey,
struct net_device *dev)
{
struct neigh_hash_table *nht = rcu_dereference_bh(tbl->nht);
struct neigh_hash_table *nht = rcu_dereference(tbl->nht);
struct neighbour *n;
u32 hash_val;

hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);
for (n = rcu_dereference(nht->hash_buckets[hash_val]);
n != NULL;
n = rcu_dereference_bh(n->next)) {
n = rcu_dereference(n->next)) {
if (n->dev == dev && key_eq(n, pkey))
return n;
}
Expand All @@ -330,6 +330,17 @@ static inline struct neighbour *__neigh_lookup_noref(struct neigh_table *tbl,
return ___neigh_lookup_noref(tbl, tbl->key_eq, tbl->hash, pkey, dev);
}

static inline void neigh_confirm(struct neighbour *n)
{
if (n) {
unsigned long now = jiffies;

/* avoid dirtying neighbour */
if (READ_ONCE(n->confirmed) != now)
WRITE_ONCE(n->confirmed, now);
}
}

void neigh_table_init(int index, struct neigh_table *tbl);
int neigh_table_clear(int index, struct neigh_table *tbl);
struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
Expand Down
23 changes: 0 additions & 23 deletions include/net/nexthop.h
Original file line number Diff line number Diff line change
Expand Up @@ -532,29 +532,6 @@ static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh)
return NULL;
}

/* Variant of nexthop_fib6_nh().
* Caller should either hold rcu_read_lock_bh(), or RTNL.
*/
static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)
{
struct nh_info *nhi;

if (nh->is_group) {
struct nh_group *nh_grp;

nh_grp = rcu_dereference_bh_rtnl(nh->nh_grp);
nh = nexthop_mpath_select(nh_grp, 0);
if (!nh)
return NULL;
}

nhi = rcu_dereference_bh_rtnl(nh->nh_info);
if (nhi->family == AF_INET6)
return &nhi->fib6_nh;

return NULL;
}

static inline struct net_device *fib6_info_nh_dev(struct fib6_info *f6i)
{
struct fib6_nh *fib6_nh;
Expand Down
9 changes: 7 additions & 2 deletions include/net/route.h
Original file line number Diff line number Diff line change
Expand Up @@ -353,10 +353,15 @@ static inline int inet_iif(const struct sk_buff *skb)
static inline int ip4_dst_hoplimit(const struct dst_entry *dst)
{
int hoplimit = dst_metric_raw(dst, RTAX_HOPLIMIT);
struct net *net = dev_net(dst->dev);

if (hoplimit == 0)
if (hoplimit == 0) {
const struct net *net;

rcu_read_lock();
net = dst_dev_net_rcu(dst);
hoplimit = READ_ONCE(net->ipv4.sysctl_ip_default_ttl);
rcu_read_unlock();
}
return hoplimit;
}

Expand Down
5 changes: 1 addition & 4 deletions include/net/sock.h
Original file line number Diff line number Diff line change
Expand Up @@ -2236,13 +2236,10 @@ static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
{
if (skb_get_dst_pending_confirm(skb)) {
struct sock *sk = skb->sk;
unsigned long now = jiffies;

/* avoid dirtying neighbour */
if (READ_ONCE(n->confirmed) != now)
WRITE_ONCE(n->confirmed, now);
if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
neigh_confirm(n);
}
}

Expand Down
4 changes: 2 additions & 2 deletions net/core/dst.c
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ void dst_dev_put(struct dst_entry *dst)
dst->ops->ifdown(dst, dev, true);
dst->input = dst_discard;
dst->output = dst_discard_out;
dst->dev = blackhole_netdev;
rcu_assign_pointer(dst->dev_rcu, blackhole_netdev);
netdev_ref_replace(dev, blackhole_netdev, &dst->dev_tracker,
GFP_ATOMIC);
}
Expand Down Expand Up @@ -257,7 +257,7 @@ unsigned int dst_blackhole_mtu(const struct dst_entry *dst)
{
unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);

return mtu ? : dst->dev->mtu;
return mtu ? : dst_dev(dst)->mtu;
}
EXPORT_SYMBOL_GPL(dst_blackhole_mtu);

Expand Down
Loading