.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
---|
1 | 2 | /* |
---|
2 | 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX |
---|
3 | 4 | * operating system. INET is implemented using the BSD Socket |
---|
.. | .. |
---|
179 | 180 | * Oh well... nobody has a sufficient solution to this |
---|
180 | 181 | * protocol bug yet. |
---|
181 | 182 | */ |
---|
182 | | - if (twsk_net(tw)->ipv4.sysctl_tcp_rfc1337 == 0) { |
---|
| 183 | + if (!READ_ONCE(twsk_net(tw)->ipv4.sysctl_tcp_rfc1337)) { |
---|
183 | 184 | kill: |
---|
184 | 185 | inet_twsk_deschedule_put(tw); |
---|
185 | 186 | return TCP_TW_SUCCESS; |
---|
.. | .. |
---|
265 | 266 | |
---|
266 | 267 | tw->tw_transparent = inet->transparent; |
---|
267 | 268 | tw->tw_mark = sk->sk_mark; |
---|
| 269 | + tw->tw_priority = sk->sk_priority; |
---|
268 | 270 | tw->tw_rcv_wscale = tp->rx_opt.rcv_wscale; |
---|
269 | 271 | tcptw->tw_rcv_nxt = tp->rcv_nxt; |
---|
270 | 272 | tcptw->tw_snd_nxt = tp->snd_nxt; |
---|
.. | .. |
---|
273 | 275 | tcptw->tw_ts_recent_stamp = tp->rx_opt.ts_recent_stamp; |
---|
274 | 276 | tcptw->tw_ts_offset = tp->tsoffset; |
---|
275 | 277 | tcptw->tw_last_oow_ack_time = 0; |
---|
276 | | - |
---|
| 278 | + tcptw->tw_tx_delay = tp->tcp_tx_delay; |
---|
277 | 279 | #if IS_ENABLED(CONFIG_IPV6) |
---|
278 | 280 | if (tw->tw_family == PF_INET6) { |
---|
279 | 281 | struct ipv6_pinfo *np = inet6_sk(sk); |
---|
.. | .. |
---|
282 | 284 | tw->tw_v6_rcv_saddr = sk->sk_v6_rcv_saddr; |
---|
283 | 285 | tw->tw_tclass = np->tclass; |
---|
284 | 286 | tw->tw_flowlabel = be32_to_cpu(np->flow_label & IPV6_FLOWLABEL_MASK); |
---|
| 287 | + tw->tw_txhash = sk->sk_txhash; |
---|
285 | 288 | tw->tw_ipv6only = sk->sk_ipv6only; |
---|
286 | 289 | } |
---|
287 | 290 | #endif |
---|
.. | .. |
---|
294 | 297 | * so the timewait ack generating code has the key. |
---|
295 | 298 | */ |
---|
296 | 299 | do { |
---|
297 | | - struct tcp_md5sig_key *key; |
---|
298 | 300 | tcptw->tw_md5_key = NULL; |
---|
299 | | - key = tp->af_specific->md5_lookup(sk, sk); |
---|
300 | | - if (key) { |
---|
301 | | - tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC); |
---|
302 | | - BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); |
---|
| 301 | + if (static_branch_unlikely(&tcp_md5_needed)) { |
---|
| 302 | + struct tcp_md5sig_key *key; |
---|
| 303 | + |
---|
| 304 | + key = tp->af_specific->md5_lookup(sk, sk); |
---|
| 305 | + if (key) { |
---|
| 306 | + tcptw->tw_md5_key = kmemdup(key, sizeof(*key), GFP_ATOMIC); |
---|
| 307 | + BUG_ON(tcptw->tw_md5_key && !tcp_alloc_md5sig_pool()); |
---|
| 308 | + } |
---|
303 | 309 | } |
---|
304 | 310 | } while (0); |
---|
305 | 311 | #endif |
---|
.. | .. |
---|
338 | 344 | void tcp_twsk_destructor(struct sock *sk) |
---|
339 | 345 | { |
---|
340 | 346 | #ifdef CONFIG_TCP_MD5SIG |
---|
341 | | - struct tcp_timewait_sock *twsk = tcp_twsk(sk); |
---|
| 347 | + if (static_branch_unlikely(&tcp_md5_needed)) { |
---|
| 348 | + struct tcp_timewait_sock *twsk = tcp_twsk(sk); |
---|
342 | 349 | |
---|
343 | | - if (twsk->tw_md5_key) |
---|
344 | | - kfree_rcu(twsk->tw_md5_key, rcu); |
---|
| 350 | + if (twsk->tw_md5_key) |
---|
| 351 | + kfree_rcu(twsk->tw_md5_key, rcu); |
---|
| 352 | + } |
---|
345 | 353 | #endif |
---|
346 | 354 | } |
---|
347 | 355 | EXPORT_SYMBOL_GPL(tcp_twsk_destructor); |
---|
.. | .. |
---|
406 | 414 | |
---|
407 | 415 | rcu_read_lock(); |
---|
408 | 416 | ca = tcp_ca_find_key(ca_key); |
---|
409 | | - if (likely(ca && try_module_get(ca->owner))) { |
---|
| 417 | + if (likely(ca && bpf_try_module_get(ca, ca->owner))) { |
---|
410 | 418 | icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst); |
---|
411 | 419 | icsk->icsk_ca_ops = ca; |
---|
412 | 420 | ca_got_dst = true; |
---|
.. | .. |
---|
417 | 425 | /* If no valid choice made yet, assign current system default ca. */ |
---|
418 | 426 | if (!ca_got_dst && |
---|
419 | 427 | (!icsk->icsk_ca_setsockopt || |
---|
420 | | - !try_module_get(icsk->icsk_ca_ops->owner))) |
---|
| 428 | + !bpf_try_module_get(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner))) |
---|
421 | 429 | tcp_assign_congestion_control(sk); |
---|
422 | 430 | |
---|
423 | 431 | tcp_set_ca_state(sk, TCP_CA_Open); |
---|
.. | .. |
---|
474 | 482 | WRITE_ONCE(newtp->rcv_nxt, seq); |
---|
475 | 483 | newtp->segs_in = 1; |
---|
476 | 484 | |
---|
477 | | - newtp->snd_sml = newtp->snd_una = |
---|
478 | | - newtp->snd_nxt = newtp->snd_up = treq->snt_isn + 1; |
---|
| 485 | + seq = treq->snt_isn + 1; |
---|
| 486 | + newtp->snd_sml = newtp->snd_una = seq; |
---|
| 487 | + WRITE_ONCE(newtp->snd_nxt, seq); |
---|
| 488 | + newtp->snd_up = seq; |
---|
479 | 489 | |
---|
480 | 490 | INIT_LIST_HEAD(&newtp->tsq_node); |
---|
481 | 491 | INIT_LIST_HEAD(&newtp->tsorted_sent_queue); |
---|
482 | 492 | |
---|
483 | 493 | tcp_init_wl(newtp, treq->rcv_isn); |
---|
484 | 494 | |
---|
485 | | - newtp->srtt_us = 0; |
---|
486 | | - newtp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT); |
---|
487 | 495 | minmax_reset(&newtp->rtt_min, tcp_jiffies32, ~0U); |
---|
488 | | - newicsk->icsk_rto = TCP_TIMEOUT_INIT; |
---|
489 | 496 | newicsk->icsk_ack.lrcvtime = tcp_jiffies32; |
---|
490 | 497 | |
---|
491 | | - newtp->packets_out = 0; |
---|
492 | | - newtp->retrans_out = 0; |
---|
493 | | - newtp->sacked_out = 0; |
---|
494 | | - newtp->snd_ssthresh = TCP_INFINITE_SSTHRESH; |
---|
495 | | - newtp->tlp_high_seq = 0; |
---|
496 | 498 | newtp->lsndtime = tcp_jiffies32; |
---|
497 | 499 | newsk->sk_txhash = treq->txhash; |
---|
498 | | - newtp->last_oow_ack_time = 0; |
---|
499 | 500 | newtp->total_retrans = req->num_retrans; |
---|
500 | | - |
---|
501 | | - /* So many TCP implementations out there (incorrectly) count the |
---|
502 | | - * initial SYN frame in their delayed-ACK and congestion control |
---|
503 | | - * algorithms that we must have the following bandaid to talk |
---|
504 | | - * efficiently to them. -DaveM |
---|
505 | | - */ |
---|
506 | | - newtp->snd_cwnd = TCP_INIT_CWND; |
---|
507 | | - newtp->snd_cwnd_cnt = 0; |
---|
508 | | - |
---|
509 | | - /* There's a bubble in the pipe until at least the first ACK. */ |
---|
510 | | - newtp->app_limited = ~0U; |
---|
511 | 501 | |
---|
512 | 502 | tcp_init_xmit_timers(newsk); |
---|
513 | 503 | WRITE_ONCE(newtp->write_seq, newtp->pushed_seq = treq->snt_isn + 1); |
---|
514 | | - |
---|
515 | | - newtp->rx_opt.saw_tstamp = 0; |
---|
516 | | - |
---|
517 | | - newtp->rx_opt.dsack = 0; |
---|
518 | | - newtp->rx_opt.num_sacks = 0; |
---|
519 | | - |
---|
520 | | - newtp->urg_data = 0; |
---|
521 | 504 | |
---|
522 | 505 | if (sock_flag(newsk, SOCK_KEEPOPEN)) |
---|
523 | 506 | inet_csk_reset_keepalive_timer(newsk, |
---|
.. | .. |
---|
547 | 530 | newtp->rx_opt.ts_recent_stamp = 0; |
---|
548 | 531 | newtp->tcp_header_len = sizeof(struct tcphdr); |
---|
549 | 532 | } |
---|
| 533 | + if (req->num_timeout) { |
---|
| 534 | + newtp->undo_marker = treq->snt_isn; |
---|
| 535 | + newtp->retrans_stamp = div_u64(treq->snt_synack, |
---|
| 536 | + USEC_PER_SEC / TCP_TS_HZ); |
---|
| 537 | + } |
---|
550 | 538 | newtp->tsoffset = treq->ts_off; |
---|
551 | 539 | #ifdef CONFIG_TCP_MD5SIG |
---|
552 | 540 | newtp->md5sig_info = NULL; /*XXX*/ |
---|
553 | | - if (newtp->af_specific->md5_lookup(sk, newsk)) |
---|
| 541 | + if (treq->af_specific->req_md5_lookup(sk, req_to_sk(req))) |
---|
554 | 542 | newtp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED; |
---|
555 | 543 | #endif |
---|
556 | 544 | if (skb->len >= TCP_MSS_DEFAULT + newtp->tcp_header_len) |
---|
.. | .. |
---|
558 | 546 | newtp->rx_opt.mss_clamp = req->mss; |
---|
559 | 547 | tcp_ecn_openreq_child(newtp, req); |
---|
560 | 548 | newtp->fastopen_req = NULL; |
---|
561 | | - newtp->fastopen_rsk = NULL; |
---|
562 | | - newtp->syn_data_acked = 0; |
---|
563 | | - newtp->rack.mstamp = 0; |
---|
564 | | - newtp->rack.advanced = 0; |
---|
565 | | - newtp->rack.reo_wnd_steps = 1; |
---|
566 | | - newtp->rack.last_delivered = 0; |
---|
567 | | - newtp->rack.reo_wnd_persist = 0; |
---|
568 | | - newtp->rack.dsack_seen = 0; |
---|
| 549 | + RCU_INIT_POINTER(newtp->fastopen_rsk, NULL); |
---|
| 550 | + |
---|
| 551 | + tcp_bpf_clone(sk, newsk); |
---|
569 | 552 | |
---|
570 | 553 | __TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS); |
---|
571 | 554 | |
---|
.. | .. |
---|
791 | 774 | if (!child) |
---|
792 | 775 | goto listen_overflow; |
---|
793 | 776 | |
---|
| 777 | + if (own_req && rsk_drop_req(req)) { |
---|
| 778 | + reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req); |
---|
| 779 | + inet_csk_reqsk_queue_drop_and_put(sk, req); |
---|
| 780 | + return child; |
---|
| 781 | + } |
---|
| 782 | + |
---|
794 | 783 | sock_rps_save_rxhash(child, skb); |
---|
795 | 784 | tcp_synack_rtt_meas(child, req); |
---|
796 | 785 | *req_stolen = !own_req; |
---|
.. | .. |
---|
839 | 828 | |
---|
840 | 829 | int tcp_child_process(struct sock *parent, struct sock *child, |
---|
841 | 830 | struct sk_buff *skb) |
---|
| 831 | + __releases(&((child)->sk_lock.slock)) |
---|
842 | 832 | { |
---|
843 | 833 | int ret = 0; |
---|
844 | 834 | int state = child->sk_state; |
---|