| .. | .. |
|---|
| 40 | 40 | |
|---|
| 41 | 41 | struct tcp_metrics_block { |
|---|
| 42 | 42 | struct tcp_metrics_block __rcu *tcpm_next; |
|---|
| 43 | | - possible_net_t tcpm_net; |
|---|
| 43 | + struct net *tcpm_net; |
|---|
| 44 | 44 | struct inetpeer_addr tcpm_saddr; |
|---|
| 45 | 45 | struct inetpeer_addr tcpm_daddr; |
|---|
| 46 | 46 | unsigned long tcpm_stamp; |
|---|
| .. | .. |
|---|
| 51 | 51 | struct rcu_head rcu_head; |
|---|
| 52 | 52 | }; |
|---|
| 53 | 53 | |
|---|
| 54 | | -static inline struct net *tm_net(struct tcp_metrics_block *tm) |
|---|
| 54 | +static inline struct net *tm_net(const struct tcp_metrics_block *tm) |
|---|
| 55 | 55 | { |
|---|
| 56 | | - return read_pnet(&tm->tcpm_net); |
|---|
| 56 | + /* Paired with the WRITE_ONCE() in tcpm_new() */ |
|---|
| 57 | + return READ_ONCE(tm->tcpm_net); |
|---|
| 57 | 58 | } |
|---|
| 58 | 59 | |
|---|
| 59 | 60 | static bool tcp_metric_locked(struct tcp_metrics_block *tm, |
|---|
| 60 | 61 | enum tcp_metric_index idx) |
|---|
| 61 | 62 | { |
|---|
| 62 | | - return tm->tcpm_lock & (1 << idx); |
|---|
| 63 | + /* Paired with WRITE_ONCE() in tcpm_suck_dst() */ |
|---|
| 64 | + return READ_ONCE(tm->tcpm_lock) & (1 << idx); |
|---|
| 63 | 65 | } |
|---|
| 64 | 66 | |
|---|
| 65 | | -static u32 tcp_metric_get(struct tcp_metrics_block *tm, |
|---|
| 67 | +static u32 tcp_metric_get(const struct tcp_metrics_block *tm, |
|---|
| 66 | 68 | enum tcp_metric_index idx) |
|---|
| 67 | 69 | { |
|---|
| 68 | | - return tm->tcpm_vals[idx]; |
|---|
| 70 | + /* Paired with WRITE_ONCE() in tcp_metric_set() */ |
|---|
| 71 | + return READ_ONCE(tm->tcpm_vals[idx]); |
|---|
| 69 | 72 | } |
|---|
| 70 | 73 | |
|---|
| 71 | 74 | static void tcp_metric_set(struct tcp_metrics_block *tm, |
|---|
| 72 | 75 | enum tcp_metric_index idx, |
|---|
| 73 | 76 | u32 val) |
|---|
| 74 | 77 | { |
|---|
| 75 | | - tm->tcpm_vals[idx] = val; |
|---|
| 78 | + /* Paired with READ_ONCE() in tcp_metric_get() */ |
|---|
| 79 | + WRITE_ONCE(tm->tcpm_vals[idx], val); |
|---|
| 76 | 80 | } |
|---|
| 77 | 81 | |
|---|
| 78 | 82 | static bool addr_same(const struct inetpeer_addr *a, |
|---|
| 79 | 83 | const struct inetpeer_addr *b) |
|---|
| 80 | 84 | { |
|---|
| 81 | | - return inetpeer_addr_cmp(a, b) == 0; |
|---|
| 85 | + return (a->family == b->family) && !inetpeer_addr_cmp(a, b); |
|---|
| 82 | 86 | } |
|---|
| 83 | 87 | |
|---|
| 84 | 88 | struct tcpm_hash_bucket { |
|---|
| .. | .. |
|---|
| 89 | 93 | static unsigned int tcp_metrics_hash_log __read_mostly; |
|---|
| 90 | 94 | |
|---|
| 91 | 95 | static DEFINE_SPINLOCK(tcp_metrics_lock); |
|---|
| 96 | +static DEFINE_SEQLOCK(fastopen_seqlock); |
|---|
| 92 | 97 | |
|---|
| 93 | 98 | static void tcpm_suck_dst(struct tcp_metrics_block *tm, |
|---|
| 94 | 99 | const struct dst_entry *dst, |
|---|
| .. | .. |
|---|
| 97 | 102 | u32 msval; |
|---|
| 98 | 103 | u32 val; |
|---|
| 99 | 104 | |
|---|
| 100 | | - tm->tcpm_stamp = jiffies; |
|---|
| 105 | + WRITE_ONCE(tm->tcpm_stamp, jiffies); |
|---|
| 101 | 106 | |
|---|
| 102 | 107 | val = 0; |
|---|
| 103 | 108 | if (dst_metric_locked(dst, RTAX_RTT)) |
|---|
| .. | .. |
|---|
| 110 | 115 | val |= 1 << TCP_METRIC_CWND; |
|---|
| 111 | 116 | if (dst_metric_locked(dst, RTAX_REORDERING)) |
|---|
| 112 | 117 | val |= 1 << TCP_METRIC_REORDERING; |
|---|
| 113 | | - tm->tcpm_lock = val; |
|---|
| 118 | + /* Paired with READ_ONCE() in tcp_metric_locked() */ |
|---|
| 119 | + WRITE_ONCE(tm->tcpm_lock, val); |
|---|
| 114 | 120 | |
|---|
| 115 | 121 | msval = dst_metric_raw(dst, RTAX_RTT); |
|---|
| 116 | | - tm->tcpm_vals[TCP_METRIC_RTT] = msval * USEC_PER_MSEC; |
|---|
| 122 | + tcp_metric_set(tm, TCP_METRIC_RTT, msval * USEC_PER_MSEC); |
|---|
| 117 | 123 | |
|---|
| 118 | 124 | msval = dst_metric_raw(dst, RTAX_RTTVAR); |
|---|
| 119 | | - tm->tcpm_vals[TCP_METRIC_RTTVAR] = msval * USEC_PER_MSEC; |
|---|
| 120 | | - tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH); |
|---|
| 121 | | - tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND); |
|---|
| 122 | | - tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING); |
|---|
| 125 | + tcp_metric_set(tm, TCP_METRIC_RTTVAR, msval * USEC_PER_MSEC); |
|---|
| 126 | + tcp_metric_set(tm, TCP_METRIC_SSTHRESH, |
|---|
| 127 | + dst_metric_raw(dst, RTAX_SSTHRESH)); |
|---|
| 128 | + tcp_metric_set(tm, TCP_METRIC_CWND, |
|---|
| 129 | + dst_metric_raw(dst, RTAX_CWND)); |
|---|
| 130 | + tcp_metric_set(tm, TCP_METRIC_REORDERING, |
|---|
| 131 | + dst_metric_raw(dst, RTAX_REORDERING)); |
|---|
| 123 | 132 | if (fastopen_clear) { |
|---|
| 133 | + write_seqlock(&fastopen_seqlock); |
|---|
| 124 | 134 | tm->tcpm_fastopen.mss = 0; |
|---|
| 125 | 135 | tm->tcpm_fastopen.syn_loss = 0; |
|---|
| 126 | 136 | tm->tcpm_fastopen.try_exp = 0; |
|---|
| 127 | 137 | tm->tcpm_fastopen.cookie.exp = false; |
|---|
| 128 | 138 | tm->tcpm_fastopen.cookie.len = 0; |
|---|
| 139 | + write_sequnlock(&fastopen_seqlock); |
|---|
| 129 | 140 | } |
|---|
| 130 | 141 | } |
|---|
| 131 | 142 | |
|---|
| 132 | 143 | #define TCP_METRICS_TIMEOUT (60 * 60 * HZ) |
|---|
| 133 | 144 | |
|---|
| 134 | | -static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst) |
|---|
| 145 | +static void tcpm_check_stamp(struct tcp_metrics_block *tm, |
|---|
| 146 | + const struct dst_entry *dst) |
|---|
| 135 | 147 | { |
|---|
| 136 | | - if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT))) |
|---|
| 148 | + unsigned long limit; |
|---|
| 149 | + |
|---|
| 150 | + if (!tm) |
|---|
| 151 | + return; |
|---|
| 152 | + limit = READ_ONCE(tm->tcpm_stamp) + TCP_METRICS_TIMEOUT; |
|---|
| 153 | + if (unlikely(time_after(jiffies, limit))) |
|---|
| 137 | 154 | tcpm_suck_dst(tm, dst, false); |
|---|
| 138 | 155 | } |
|---|
| 139 | 156 | |
|---|
| .. | .. |
|---|
| 174 | 191 | oldest = deref_locked(tcp_metrics_hash[hash].chain); |
|---|
| 175 | 192 | for (tm = deref_locked(oldest->tcpm_next); tm; |
|---|
| 176 | 193 | tm = deref_locked(tm->tcpm_next)) { |
|---|
| 177 | | - if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp)) |
|---|
| 194 | + if (time_before(READ_ONCE(tm->tcpm_stamp), |
|---|
| 195 | + READ_ONCE(oldest->tcpm_stamp))) |
|---|
| 178 | 196 | oldest = tm; |
|---|
| 179 | 197 | } |
|---|
| 180 | 198 | tm = oldest; |
|---|
| 181 | 199 | } else { |
|---|
| 182 | | - tm = kmalloc(sizeof(*tm), GFP_ATOMIC); |
|---|
| 200 | + tm = kzalloc(sizeof(*tm), GFP_ATOMIC); |
|---|
| 183 | 201 | if (!tm) |
|---|
| 184 | 202 | goto out_unlock; |
|---|
| 185 | 203 | } |
|---|
| 186 | | - write_pnet(&tm->tcpm_net, net); |
|---|
| 204 | + /* Paired with the READ_ONCE() in tm_net() */ |
|---|
| 205 | + WRITE_ONCE(tm->tcpm_net, net); |
|---|
| 206 | + |
|---|
| 187 | 207 | tm->tcpm_saddr = *saddr; |
|---|
| 188 | 208 | tm->tcpm_daddr = *daddr; |
|---|
| 189 | 209 | |
|---|
| 190 | | - tcpm_suck_dst(tm, dst, true); |
|---|
| 210 | + tcpm_suck_dst(tm, dst, reclaim); |
|---|
| 191 | 211 | |
|---|
| 192 | 212 | if (likely(!reclaim)) { |
|---|
| 193 | 213 | tm->tcpm_next = tcp_metrics_hash[hash].chain; |
|---|
| .. | .. |
|---|
| 434 | 454 | tp->reordering); |
|---|
| 435 | 455 | } |
|---|
| 436 | 456 | } |
|---|
| 437 | | - tm->tcpm_stamp = jiffies; |
|---|
| 457 | + WRITE_ONCE(tm->tcpm_stamp, jiffies); |
|---|
| 438 | 458 | out_unlock: |
|---|
| 439 | 459 | rcu_read_unlock(); |
|---|
| 440 | 460 | } |
|---|
| .. | .. |
|---|
| 538 | 558 | |
|---|
| 539 | 559 | return ret; |
|---|
| 540 | 560 | } |
|---|
| 541 | | - |
|---|
| 542 | | -static DEFINE_SEQLOCK(fastopen_seqlock); |
|---|
| 543 | 561 | |
|---|
| 544 | 562 | void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, |
|---|
| 545 | 563 | struct tcp_fastopen_cookie *cookie) |
|---|
| .. | .. |
|---|
| 647 | 665 | } |
|---|
| 648 | 666 | |
|---|
| 649 | 667 | if (nla_put_msecs(msg, TCP_METRICS_ATTR_AGE, |
|---|
| 650 | | - jiffies - tm->tcpm_stamp, |
|---|
| 668 | + jiffies - READ_ONCE(tm->tcpm_stamp), |
|---|
| 651 | 669 | TCP_METRICS_ATTR_PAD) < 0) |
|---|
| 652 | 670 | goto nla_put_failure; |
|---|
| 653 | 671 | |
|---|
| .. | .. |
|---|
| 658 | 676 | if (!nest) |
|---|
| 659 | 677 | goto nla_put_failure; |
|---|
| 660 | 678 | for (i = 0; i < TCP_METRIC_MAX_KERNEL + 1; i++) { |
|---|
| 661 | | - u32 val = tm->tcpm_vals[i]; |
|---|
| 679 | + u32 val = tcp_metric_get(tm, i); |
|---|
| 662 | 680 | |
|---|
| 663 | 681 | if (!val) |
|---|
| 664 | 682 | continue; |
|---|