| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX |
|---|
| 3 | 4 | * operating system. INET is implemented using the BSD Socket |
|---|
| .. | .. |
|---|
| 58 | 59 | * Some other random speedups. |
|---|
| 59 | 60 | * Cyrus Durgin : Cleaned up file for kmod hacks. |
|---|
| 60 | 61 | * Andi Kleen : Fix inet_stream_connect TCP race. |
|---|
| 61 | | - * |
|---|
| 62 | | - * This program is free software; you can redistribute it and/or |
|---|
| 63 | | - * modify it under the terms of the GNU General Public License |
|---|
| 64 | | - * as published by the Free Software Foundation; either version |
|---|
| 65 | | - * 2 of the License, or (at your option) any later version. |
|---|
| 66 | 62 | */ |
|---|
| 67 | 63 | |
|---|
| 68 | 64 | #define pr_fmt(fmt) "IPv4: " fmt |
|---|
| .. | .. |
|---|
| 120 | 116 | #include <linux/mroute.h> |
|---|
| 121 | 117 | #endif |
|---|
| 122 | 118 | #include <net/l3mdev.h> |
|---|
| 119 | +#include <net/compat.h> |
|---|
| 123 | 120 | |
|---|
| 124 | 121 | #include <trace/events/sock.h> |
|---|
| 125 | 122 | |
|---|
| .. | .. |
|---|
| 136 | 133 | struct inet_sock *inet = inet_sk(sk); |
|---|
| 137 | 134 | |
|---|
| 138 | 135 | __skb_queue_purge(&sk->sk_receive_queue); |
|---|
| 136 | + if (sk->sk_rx_skb_cache) { |
|---|
| 137 | + __kfree_skb(sk->sk_rx_skb_cache); |
|---|
| 138 | + sk->sk_rx_skb_cache = NULL; |
|---|
| 139 | + } |
|---|
| 139 | 140 | __skb_queue_purge(&sk->sk_error_queue); |
|---|
| 140 | 141 | |
|---|
| 141 | 142 | sk_mem_reclaim(sk); |
|---|
| .. | .. |
|---|
| 156 | 157 | WARN_ON(sk->sk_forward_alloc); |
|---|
| 157 | 158 | |
|---|
| 158 | 159 | kfree(rcu_dereference_protected(inet->inet_opt, 1)); |
|---|
| 159 | | - dst_release(rcu_dereference_check(sk->sk_dst_cache, 1)); |
|---|
| 160 | | - dst_release(sk->sk_rx_dst); |
|---|
| 160 | + dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1)); |
|---|
| 161 | + dst_release(rcu_dereference_protected(sk->sk_rx_dst, 1)); |
|---|
| 161 | 162 | sk_refcnt_debug_dec(sk); |
|---|
| 162 | 163 | } |
|---|
| 163 | 164 | EXPORT_SYMBOL(inet_sock_destruct); |
|---|
| .. | .. |
|---|
| 208 | 209 | if (!((1 << old_state) & (TCPF_CLOSE | TCPF_LISTEN))) |
|---|
| 209 | 210 | goto out; |
|---|
| 210 | 211 | |
|---|
| 212 | + WRITE_ONCE(sk->sk_max_ack_backlog, backlog); |
|---|
| 211 | 213 | /* Really, if the socket is already in listen state |
|---|
| 212 | 214 | * we can only allow the backlog to be adjusted. |
|---|
| 213 | 215 | */ |
|---|
| .. | .. |
|---|
| 218 | 220 | * because the socket was in TCP_LISTEN state previously but |
|---|
| 219 | 221 | * was shutdown() rather than close(). |
|---|
| 220 | 222 | */ |
|---|
| 221 | | - tcp_fastopen = sock_net(sk)->ipv4.sysctl_tcp_fastopen; |
|---|
| 223 | + tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen); |
|---|
| 222 | 224 | if ((tcp_fastopen & TFO_SERVER_WO_SOCKOPT1) && |
|---|
| 223 | 225 | (tcp_fastopen & TFO_SERVER_ENABLE) && |
|---|
| 224 | 226 | !inet_csk(sk)->icsk_accept_queue.fastopenq.max_qlen) { |
|---|
| .. | .. |
|---|
| 231 | 233 | goto out; |
|---|
| 232 | 234 | tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); |
|---|
| 233 | 235 | } |
|---|
| 234 | | - sk->sk_max_ack_backlog = backlog; |
|---|
| 235 | 236 | err = 0; |
|---|
| 236 | 237 | |
|---|
| 237 | 238 | out: |
|---|
| .. | .. |
|---|
| 337 | 338 | inet->hdrincl = 1; |
|---|
| 338 | 339 | } |
|---|
| 339 | 340 | |
|---|
| 340 | | - if (net->ipv4.sysctl_ip_no_pmtu_disc) |
|---|
| 341 | + if (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) |
|---|
| 341 | 342 | inet->pmtudisc = IP_PMTUDISC_DONT; |
|---|
| 342 | 343 | else |
|---|
| 343 | 344 | inet->pmtudisc = IP_PMTUDISC_WANT; |
|---|
| .. | .. |
|---|
| 410 | 411 | if (sk) { |
|---|
| 411 | 412 | long timeout; |
|---|
| 412 | 413 | |
|---|
| 414 | + if (!sk->sk_kern_sock) |
|---|
| 415 | + BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk); |
|---|
| 416 | + |
|---|
| 413 | 417 | /* Applications forget to leave groups before exiting */ |
|---|
| 414 | 418 | ip_mc_drop_socket(sk); |
|---|
| 415 | 419 | |
|---|
| .. | .. |
|---|
| 450 | 454 | if (err) |
|---|
| 451 | 455 | return err; |
|---|
| 452 | 456 | |
|---|
| 453 | | - return __inet_bind(sk, uaddr, addr_len, false, true); |
|---|
| 457 | + return __inet_bind(sk, uaddr, addr_len, BIND_WITH_LOCK); |
|---|
| 454 | 458 | } |
|---|
| 455 | 459 | EXPORT_SYMBOL(inet_bind); |
|---|
| 456 | 460 | |
|---|
| 457 | 461 | int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, |
|---|
| 458 | | - bool force_bind_address_no_port, bool with_lock) |
|---|
| 462 | + u32 flags) |
|---|
| 459 | 463 | { |
|---|
| 460 | 464 | struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; |
|---|
| 461 | 465 | struct inet_sock *inet = inet_sk(sk); |
|---|
| .. | .. |
|---|
| 494 | 498 | goto out; |
|---|
| 495 | 499 | |
|---|
| 496 | 500 | snum = ntohs(addr->sin_port); |
|---|
| 501 | + err = -EPERM; |
|---|
| 502 | + if (snum && inet_is_local_unbindable_port(net, snum)) |
|---|
| 503 | + goto out; |
|---|
| 504 | + |
|---|
| 497 | 505 | err = -EACCES; |
|---|
| 498 | | - if (snum && snum < inet_prot_sock(net) && |
|---|
| 506 | + if (snum && inet_port_requires_bind_service(net, snum) && |
|---|
| 499 | 507 | !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE)) |
|---|
| 500 | 508 | goto out; |
|---|
| 501 | 509 | |
|---|
| .. | .. |
|---|
| 506 | 514 | * would be illegal to use them (multicast/broadcast) in |
|---|
| 507 | 515 | * which case the sending device address is used. |
|---|
| 508 | 516 | */ |
|---|
| 509 | | - if (with_lock) |
|---|
| 517 | + if (flags & BIND_WITH_LOCK) |
|---|
| 510 | 518 | lock_sock(sk); |
|---|
| 511 | 519 | |
|---|
| 512 | 520 | /* Check these errors (active socket, double bind). */ |
|---|
| .. | .. |
|---|
| 520 | 528 | |
|---|
| 521 | 529 | /* Make sure we are allowed to bind here. */ |
|---|
| 522 | 530 | if (snum || !(inet->bind_address_no_port || |
|---|
| 523 | | - force_bind_address_no_port)) { |
|---|
| 531 | + (flags & BIND_FORCE_ADDRESS_NO_PORT))) { |
|---|
| 524 | 532 | if (sk->sk_prot->get_port(sk, snum)) { |
|---|
| 525 | 533 | inet->inet_saddr = inet->inet_rcv_saddr = 0; |
|---|
| 526 | 534 | err = -EADDRINUSE; |
|---|
| 527 | 535 | goto out_release_sock; |
|---|
| 528 | 536 | } |
|---|
| 529 | | - err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk); |
|---|
| 530 | | - if (err) { |
|---|
| 531 | | - inet->inet_saddr = inet->inet_rcv_saddr = 0; |
|---|
| 532 | | - goto out_release_sock; |
|---|
| 537 | + if (!(flags & BIND_FROM_BPF)) { |
|---|
| 538 | + err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk); |
|---|
| 539 | + if (err) { |
|---|
| 540 | + inet->inet_saddr = inet->inet_rcv_saddr = 0; |
|---|
| 541 | + goto out_release_sock; |
|---|
| 542 | + } |
|---|
| 533 | 543 | } |
|---|
| 534 | 544 | } |
|---|
| 535 | 545 | |
|---|
| .. | .. |
|---|
| 543 | 553 | sk_dst_reset(sk); |
|---|
| 544 | 554 | err = 0; |
|---|
| 545 | 555 | out_release_sock: |
|---|
| 546 | | - if (with_lock) |
|---|
| 556 | + if (flags & BIND_WITH_LOCK) |
|---|
| 547 | 557 | release_sock(sk); |
|---|
| 548 | 558 | out: |
|---|
| 549 | 559 | return err; |
|---|
| .. | .. |
|---|
| 566 | 576 | return err; |
|---|
| 567 | 577 | } |
|---|
| 568 | 578 | |
|---|
| 569 | | - if (!inet_sk(sk)->inet_num && inet_autobind(sk)) |
|---|
| 579 | + if (data_race(!inet_sk(sk)->inet_num) && inet_autobind(sk)) |
|---|
| 570 | 580 | return -EAGAIN; |
|---|
| 571 | 581 | return sk->sk_prot->connect(sk, uaddr, addr_len); |
|---|
| 572 | 582 | } |
|---|
| .. | .. |
|---|
| 753 | 763 | } |
|---|
| 754 | 764 | EXPORT_SYMBOL(inet_accept); |
|---|
| 755 | 765 | |
|---|
| 756 | | - |
|---|
| 757 | 766 | /* |
|---|
| 758 | 767 | * This does both peername and sockname. |
|---|
| 759 | 768 | */ |
|---|
| 760 | 769 | int inet_getname(struct socket *sock, struct sockaddr *uaddr, |
|---|
| 761 | | - int peer) |
|---|
| 770 | + int peer) |
|---|
| 762 | 771 | { |
|---|
| 763 | 772 | struct sock *sk = sock->sk; |
|---|
| 764 | 773 | struct inet_sock *inet = inet_sk(sk); |
|---|
| .. | .. |
|---|
| 779 | 788 | sin->sin_port = inet->inet_sport; |
|---|
| 780 | 789 | sin->sin_addr.s_addr = addr; |
|---|
| 781 | 790 | } |
|---|
| 791 | + if (cgroup_bpf_enabled) |
|---|
| 792 | + BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, |
|---|
| 793 | + peer ? BPF_CGROUP_INET4_GETPEERNAME : |
|---|
| 794 | + BPF_CGROUP_INET4_GETSOCKNAME, |
|---|
| 795 | + NULL); |
|---|
| 782 | 796 | memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); |
|---|
| 783 | 797 | return sizeof(*sin); |
|---|
| 784 | 798 | } |
|---|
| 785 | 799 | EXPORT_SYMBOL(inet_getname); |
|---|
| 786 | 800 | |
|---|
| 801 | +int inet_send_prepare(struct sock *sk) |
|---|
| 802 | +{ |
|---|
| 803 | + sock_rps_record_flow(sk); |
|---|
| 804 | + |
|---|
| 805 | + /* We may need to bind the socket. */ |
|---|
| 806 | + if (data_race(!inet_sk(sk)->inet_num) && !sk->sk_prot->no_autobind && |
|---|
| 807 | + inet_autobind(sk)) |
|---|
| 808 | + return -EAGAIN; |
|---|
| 809 | + |
|---|
| 810 | + return 0; |
|---|
| 811 | +} |
|---|
| 812 | +EXPORT_SYMBOL_GPL(inet_send_prepare); |
|---|
| 813 | + |
|---|
| 787 | 814 | int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) |
|---|
| 788 | 815 | { |
|---|
| 789 | 816 | struct sock *sk = sock->sk; |
|---|
| 790 | 817 | |
|---|
| 791 | | - sock_rps_record_flow(sk); |
|---|
| 792 | | - |
|---|
| 793 | | - /* We may need to bind the socket. */ |
|---|
| 794 | | - if (!inet_sk(sk)->inet_num && !sk->sk_prot->no_autobind && |
|---|
| 795 | | - inet_autobind(sk)) |
|---|
| 818 | + if (unlikely(inet_send_prepare(sk))) |
|---|
| 796 | 819 | return -EAGAIN; |
|---|
| 797 | 820 | |
|---|
| 798 | | - return sk->sk_prot->sendmsg(sk, msg, size); |
|---|
| 821 | + return INDIRECT_CALL_2(sk->sk_prot->sendmsg, tcp_sendmsg, udp_sendmsg, |
|---|
| 822 | + sk, msg, size); |
|---|
| 799 | 823 | } |
|---|
| 800 | 824 | EXPORT_SYMBOL(inet_sendmsg); |
|---|
| 801 | 825 | |
|---|
| .. | .. |
|---|
| 804 | 828 | { |
|---|
| 805 | 829 | struct sock *sk = sock->sk; |
|---|
| 806 | 830 | |
|---|
| 807 | | - sock_rps_record_flow(sk); |
|---|
| 808 | | - |
|---|
| 809 | | - /* We may need to bind the socket. */ |
|---|
| 810 | | - if (!inet_sk(sk)->inet_num && !sk->sk_prot->no_autobind && |
|---|
| 811 | | - inet_autobind(sk)) |
|---|
| 831 | + if (unlikely(inet_send_prepare(sk))) |
|---|
| 812 | 832 | return -EAGAIN; |
|---|
| 813 | 833 | |
|---|
| 814 | 834 | if (sk->sk_prot->sendpage) |
|---|
| .. | .. |
|---|
| 817 | 837 | } |
|---|
| 818 | 838 | EXPORT_SYMBOL(inet_sendpage); |
|---|
| 819 | 839 | |
|---|
| 840 | +INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *, struct msghdr *, |
|---|
| 841 | + size_t, int, int, int *)); |
|---|
| 820 | 842 | int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, |
|---|
| 821 | 843 | int flags) |
|---|
| 822 | 844 | { |
|---|
| .. | .. |
|---|
| 827 | 849 | if (likely(!(flags & MSG_ERRQUEUE))) |
|---|
| 828 | 850 | sock_rps_record_flow(sk); |
|---|
| 829 | 851 | |
|---|
| 830 | | - err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, |
|---|
| 831 | | - flags & ~MSG_DONTWAIT, &addr_len); |
|---|
| 852 | + err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg, |
|---|
| 853 | + sk, msg, size, flags & MSG_DONTWAIT, |
|---|
| 854 | + flags & ~MSG_DONTWAIT, &addr_len); |
|---|
| 832 | 855 | if (err >= 0) |
|---|
| 833 | 856 | msg->msg_namelen = addr_len; |
|---|
| 834 | 857 | return err; |
|---|
| .. | .. |
|---|
| 863 | 886 | err = -ENOTCONN; |
|---|
| 864 | 887 | /* Hack to wake up other listeners, who can poll for |
|---|
| 865 | 888 | EPOLLHUP, even on eg. unconnected UDP sockets -- RR */ |
|---|
| 866 | | - /* fall through */ |
|---|
| 889 | + fallthrough; |
|---|
| 867 | 890 | default: |
|---|
| 868 | | - sk->sk_shutdown |= how; |
|---|
| 891 | + WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | how); |
|---|
| 869 | 892 | if (sk->sk_prot->shutdown) |
|---|
| 870 | 893 | sk->sk_prot->shutdown(sk, how); |
|---|
| 871 | 894 | break; |
|---|
| .. | .. |
|---|
| 877 | 900 | case TCP_LISTEN: |
|---|
| 878 | 901 | if (!(how & RCV_SHUTDOWN)) |
|---|
| 879 | 902 | break; |
|---|
| 880 | | - /* fall through */ |
|---|
| 903 | + fallthrough; |
|---|
| 881 | 904 | case TCP_SYN_SENT: |
|---|
| 882 | 905 | err = sk->sk_prot->disconnect(sk, O_NONBLOCK); |
|---|
| 883 | 906 | sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED; |
|---|
| .. | .. |
|---|
| 911 | 934 | struct rtentry rt; |
|---|
| 912 | 935 | |
|---|
| 913 | 936 | switch (cmd) { |
|---|
| 914 | | - case SIOCGSTAMP: |
|---|
| 915 | | - err = sock_get_timestamp(sk, (struct timeval __user *)arg); |
|---|
| 916 | | - break; |
|---|
| 917 | | - case SIOCGSTAMPNS: |
|---|
| 918 | | - err = sock_get_timestampns(sk, (struct timespec __user *)arg); |
|---|
| 919 | | - break; |
|---|
| 920 | 937 | case SIOCADDRT: |
|---|
| 921 | 938 | case SIOCDELRT: |
|---|
| 922 | 939 | if (copy_from_user(&rt, p, sizeof(struct rtentry))) |
|---|
| .. | .. |
|---|
| 965 | 982 | EXPORT_SYMBOL(inet_ioctl); |
|---|
| 966 | 983 | |
|---|
| 967 | 984 | #ifdef CONFIG_COMPAT |
|---|
| 985 | +static int inet_compat_routing_ioctl(struct sock *sk, unsigned int cmd, |
|---|
| 986 | + struct compat_rtentry __user *ur) |
|---|
| 987 | +{ |
|---|
| 988 | + compat_uptr_t rtdev; |
|---|
| 989 | + struct rtentry rt; |
|---|
| 990 | + |
|---|
| 991 | + if (copy_from_user(&rt.rt_dst, &ur->rt_dst, |
|---|
| 992 | + 3 * sizeof(struct sockaddr)) || |
|---|
| 993 | + get_user(rt.rt_flags, &ur->rt_flags) || |
|---|
| 994 | + get_user(rt.rt_metric, &ur->rt_metric) || |
|---|
| 995 | + get_user(rt.rt_mtu, &ur->rt_mtu) || |
|---|
| 996 | + get_user(rt.rt_window, &ur->rt_window) || |
|---|
| 997 | + get_user(rt.rt_irtt, &ur->rt_irtt) || |
|---|
| 998 | + get_user(rtdev, &ur->rt_dev)) |
|---|
| 999 | + return -EFAULT; |
|---|
| 1000 | + |
|---|
| 1001 | + rt.rt_dev = compat_ptr(rtdev); |
|---|
| 1002 | + return ip_rt_ioctl(sock_net(sk), cmd, &rt); |
|---|
| 1003 | +} |
|---|
| 1004 | + |
|---|
| 968 | 1005 | static int inet_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) |
|---|
| 969 | 1006 | { |
|---|
| 1007 | + void __user *argp = compat_ptr(arg); |
|---|
| 970 | 1008 | struct sock *sk = sock->sk; |
|---|
| 971 | | - int err = -ENOIOCTLCMD; |
|---|
| 972 | 1009 | |
|---|
| 973 | | - if (sk->sk_prot->compat_ioctl) |
|---|
| 974 | | - err = sk->sk_prot->compat_ioctl(sk, cmd, arg); |
|---|
| 975 | | - |
|---|
| 976 | | - return err; |
|---|
| 1010 | + switch (cmd) { |
|---|
| 1011 | + case SIOCADDRT: |
|---|
| 1012 | + case SIOCDELRT: |
|---|
| 1013 | + return inet_compat_routing_ioctl(sk, cmd, argp); |
|---|
| 1014 | + default: |
|---|
| 1015 | + if (!sk->sk_prot->compat_ioctl) |
|---|
| 1016 | + return -ENOIOCTLCMD; |
|---|
| 1017 | + return sk->sk_prot->compat_ioctl(sk, cmd, arg); |
|---|
| 1018 | + } |
|---|
| 977 | 1019 | } |
|---|
| 978 | | -#endif |
|---|
| 1020 | +#endif /* CONFIG_COMPAT */ |
|---|
| 979 | 1021 | |
|---|
| 980 | 1022 | const struct proto_ops inet_stream_ops = { |
|---|
| 981 | 1023 | .family = PF_INET, |
|---|
| .. | .. |
|---|
| 988 | 1030 | .getname = inet_getname, |
|---|
| 989 | 1031 | .poll = tcp_poll, |
|---|
| 990 | 1032 | .ioctl = inet_ioctl, |
|---|
| 1033 | + .gettstamp = sock_gettstamp, |
|---|
| 991 | 1034 | .listen = inet_listen, |
|---|
| 992 | 1035 | .shutdown = inet_shutdown, |
|---|
| 993 | 1036 | .setsockopt = sock_common_setsockopt, |
|---|
| .. | .. |
|---|
| 1004 | 1047 | .sendpage_locked = tcp_sendpage_locked, |
|---|
| 1005 | 1048 | .peek_len = tcp_peek_len, |
|---|
| 1006 | 1049 | #ifdef CONFIG_COMPAT |
|---|
| 1007 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
|---|
| 1008 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
|---|
| 1009 | 1050 | .compat_ioctl = inet_compat_ioctl, |
|---|
| 1010 | 1051 | #endif |
|---|
| 1011 | 1052 | .set_rcvlowat = tcp_set_rcvlowat, |
|---|
| .. | .. |
|---|
| 1023 | 1064 | .getname = inet_getname, |
|---|
| 1024 | 1065 | .poll = udp_poll, |
|---|
| 1025 | 1066 | .ioctl = inet_ioctl, |
|---|
| 1067 | + .gettstamp = sock_gettstamp, |
|---|
| 1026 | 1068 | .listen = sock_no_listen, |
|---|
| 1027 | 1069 | .shutdown = inet_shutdown, |
|---|
| 1028 | 1070 | .setsockopt = sock_common_setsockopt, |
|---|
| .. | .. |
|---|
| 1033 | 1075 | .sendpage = inet_sendpage, |
|---|
| 1034 | 1076 | .set_peek_off = sk_set_peek_off, |
|---|
| 1035 | 1077 | #ifdef CONFIG_COMPAT |
|---|
| 1036 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
|---|
| 1037 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
|---|
| 1038 | 1078 | .compat_ioctl = inet_compat_ioctl, |
|---|
| 1039 | 1079 | #endif |
|---|
| 1040 | 1080 | }; |
|---|
| .. | .. |
|---|
| 1055 | 1095 | .getname = inet_getname, |
|---|
| 1056 | 1096 | .poll = datagram_poll, |
|---|
| 1057 | 1097 | .ioctl = inet_ioctl, |
|---|
| 1098 | + .gettstamp = sock_gettstamp, |
|---|
| 1058 | 1099 | .listen = sock_no_listen, |
|---|
| 1059 | 1100 | .shutdown = inet_shutdown, |
|---|
| 1060 | 1101 | .setsockopt = sock_common_setsockopt, |
|---|
| .. | .. |
|---|
| 1064 | 1105 | .mmap = sock_no_mmap, |
|---|
| 1065 | 1106 | .sendpage = inet_sendpage, |
|---|
| 1066 | 1107 | #ifdef CONFIG_COMPAT |
|---|
| 1067 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
|---|
| 1068 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
|---|
| 1069 | 1108 | .compat_ioctl = inet_compat_ioctl, |
|---|
| 1070 | 1109 | #endif |
|---|
| 1071 | 1110 | }; |
|---|
| .. | .. |
|---|
| 1209 | 1248 | if (new_saddr == old_saddr) |
|---|
| 1210 | 1249 | return 0; |
|---|
| 1211 | 1250 | |
|---|
| 1212 | | - if (sock_net(sk)->ipv4.sysctl_ip_dynaddr > 1) { |
|---|
| 1251 | + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) > 1) { |
|---|
| 1213 | 1252 | pr_info("%s(): shifting inet->saddr from %pI4 to %pI4\n", |
|---|
| 1214 | 1253 | __func__, &old_saddr, &new_saddr); |
|---|
| 1215 | 1254 | } |
|---|
| .. | .. |
|---|
| 1264 | 1303 | * Other protocols have to map its equivalent state to TCP_SYN_SENT. |
|---|
| 1265 | 1304 | * DCCP maps its DCCP_REQUESTING state to TCP_SYN_SENT. -acme |
|---|
| 1266 | 1305 | */ |
|---|
| 1267 | | - if (!sock_net(sk)->ipv4.sysctl_ip_dynaddr || |
|---|
| 1306 | + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) || |
|---|
| 1268 | 1307 | sk->sk_state != TCP_SYN_SENT || |
|---|
| 1269 | 1308 | (sk->sk_userlocks & SOCK_BINDADDR_LOCK) || |
|---|
| 1270 | 1309 | (err = inet_sk_reselect_saddr(sk)) != 0) |
|---|
| .. | .. |
|---|
| 1388 | 1427 | } |
|---|
| 1389 | 1428 | EXPORT_SYMBOL(inet_gso_segment); |
|---|
| 1390 | 1429 | |
|---|
| 1430 | +static struct sk_buff *ipip_gso_segment(struct sk_buff *skb, |
|---|
| 1431 | + netdev_features_t features) |
|---|
| 1432 | +{ |
|---|
| 1433 | + if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4)) |
|---|
| 1434 | + return ERR_PTR(-EINVAL); |
|---|
| 1435 | + |
|---|
| 1436 | + return inet_gso_segment(skb, features); |
|---|
| 1437 | +} |
|---|
| 1438 | + |
|---|
| 1391 | 1439 | struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) |
|---|
| 1392 | 1440 | { |
|---|
| 1393 | 1441 | const struct net_offload *ops; |
|---|
| .. | .. |
|---|
| 1497 | 1545 | skb_gro_pull(skb, sizeof(*iph)); |
|---|
| 1498 | 1546 | skb_set_transport_header(skb, skb_gro_offset(skb)); |
|---|
| 1499 | 1547 | |
|---|
| 1500 | | - pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); |
|---|
| 1548 | + pp = indirect_call_gro_receive(tcp4_gro_receive, udp4_gro_receive, |
|---|
| 1549 | + ops->callbacks.gro_receive, head, skb); |
|---|
| 1501 | 1550 | |
|---|
| 1502 | 1551 | out_unlock: |
|---|
| 1503 | 1552 | rcu_read_unlock(); |
|---|
| .. | .. |
|---|
| 1584 | 1633 | * because any hdr with option will have been flushed in |
|---|
| 1585 | 1634 | * inet_gro_receive(). |
|---|
| 1586 | 1635 | */ |
|---|
| 1587 | | - err = ops->callbacks.gro_complete(skb, nhoff + sizeof(*iph)); |
|---|
| 1636 | + err = INDIRECT_CALL_2(ops->callbacks.gro_complete, |
|---|
| 1637 | + tcp4_gro_complete, udp4_gro_complete, |
|---|
| 1638 | + skb, nhoff + sizeof(*iph)); |
|---|
| 1588 | 1639 | |
|---|
| 1589 | 1640 | out_unlock: |
|---|
| 1590 | 1641 | rcu_read_unlock(); |
|---|
| .. | .. |
|---|
| 1678 | 1729 | }; |
|---|
| 1679 | 1730 | #endif |
|---|
| 1680 | 1731 | |
|---|
| 1681 | | -/* thinking of making this const? Don't. |
|---|
| 1682 | | - * early_demux can change based on sysctl. |
|---|
| 1683 | | - */ |
|---|
| 1684 | | -static struct net_protocol tcp_protocol = { |
|---|
| 1685 | | - .early_demux = tcp_v4_early_demux, |
|---|
| 1686 | | - .early_demux_handler = tcp_v4_early_demux, |
|---|
| 1732 | +static const struct net_protocol tcp_protocol = { |
|---|
| 1687 | 1733 | .handler = tcp_v4_rcv, |
|---|
| 1688 | 1734 | .err_handler = tcp_v4_err, |
|---|
| 1689 | 1735 | .no_policy = 1, |
|---|
| .. | .. |
|---|
| 1691 | 1737 | .icmp_strict_tag_validation = 1, |
|---|
| 1692 | 1738 | }; |
|---|
| 1693 | 1739 | |
|---|
| 1694 | | -/* thinking of making this const? Don't. |
|---|
| 1695 | | - * early_demux can change based on sysctl. |
|---|
| 1696 | | - */ |
|---|
| 1697 | | -static struct net_protocol udp_protocol = { |
|---|
| 1698 | | - .early_demux = udp_v4_early_demux, |
|---|
| 1699 | | - .early_demux_handler = udp_v4_early_demux, |
|---|
| 1740 | +static const struct net_protocol udp_protocol = { |
|---|
| 1700 | 1741 | .handler = udp_rcv, |
|---|
| 1701 | 1742 | .err_handler = udp_err, |
|---|
| 1702 | 1743 | .no_policy = 1, |
|---|
| .. | .. |
|---|
| 1772 | 1813 | free_percpu(net->mib.net_statistics); |
|---|
| 1773 | 1814 | free_percpu(net->mib.ip_statistics); |
|---|
| 1774 | 1815 | free_percpu(net->mib.tcp_statistics); |
|---|
| 1816 | +#ifdef CONFIG_MPTCP |
|---|
| 1817 | + /* allocated on demand, see mptcp_init_sock() */ |
|---|
| 1818 | + free_percpu(net->mib.mptcp_statistics); |
|---|
| 1819 | +#endif |
|---|
| 1775 | 1820 | } |
|---|
| 1776 | 1821 | |
|---|
| 1777 | 1822 | static __net_initdata struct pernet_operations ipv4_mib_ops = { |
|---|
| .. | .. |
|---|
| 1810 | 1855 | net->ipv4.sysctl_ip_early_demux = 1; |
|---|
| 1811 | 1856 | net->ipv4.sysctl_udp_early_demux = 1; |
|---|
| 1812 | 1857 | net->ipv4.sysctl_tcp_early_demux = 1; |
|---|
| 1858 | + net->ipv4.sysctl_nexthop_compat_mode = 1; |
|---|
| 1813 | 1859 | #ifdef CONFIG_SYSCTL |
|---|
| 1814 | 1860 | net->ipv4.sysctl_ip_prot_sock = PROT_SOCK; |
|---|
| 1815 | 1861 | #endif |
|---|
| .. | .. |
|---|
| 1824 | 1870 | return 0; |
|---|
| 1825 | 1871 | } |
|---|
| 1826 | 1872 | |
|---|
| 1827 | | -static __net_exit void inet_exit_net(struct net *net) |
|---|
| 1828 | | -{ |
|---|
| 1829 | | -} |
|---|
| 1830 | | - |
|---|
| 1831 | 1873 | static __net_initdata struct pernet_operations af_inet_ops = { |
|---|
| 1832 | 1874 | .init = inet_init_net, |
|---|
| 1833 | | - .exit = inet_exit_net, |
|---|
| 1834 | 1875 | }; |
|---|
| 1835 | 1876 | |
|---|
| 1836 | 1877 | static int __init init_inet_pernet_ops(void) |
|---|
| .. | .. |
|---|
| 1855 | 1896 | |
|---|
| 1856 | 1897 | static const struct net_offload ipip_offload = { |
|---|
| 1857 | 1898 | .callbacks = { |
|---|
| 1858 | | - .gso_segment = inet_gso_segment, |
|---|
| 1899 | + .gso_segment = ipip_gso_segment, |
|---|
| 1859 | 1900 | .gro_receive = ipip_gro_receive, |
|---|
| 1860 | 1901 | .gro_complete = ipip_gro_complete, |
|---|
| 1861 | 1902 | }, |
|---|
| .. | .. |
|---|
| 1894 | 1935 | { |
|---|
| 1895 | 1936 | struct inet_protosw *q; |
|---|
| 1896 | 1937 | struct list_head *r; |
|---|
| 1897 | | - int rc = -EINVAL; |
|---|
| 1938 | + int rc; |
|---|
| 1898 | 1939 | |
|---|
| 1899 | 1940 | sock_skb_cb_check_size(sizeof(struct inet_skb_parm)); |
|---|
| 1900 | 1941 | |
|---|
| .. | .. |
|---|
| 1971 | 2012 | /* Add UDP-Lite (RFC 3828) */ |
|---|
| 1972 | 2013 | udplite4_register(); |
|---|
| 1973 | 2014 | |
|---|
| 2015 | + raw_init(); |
|---|
| 2016 | + |
|---|
| 1974 | 2017 | ping_init(); |
|---|
| 1975 | 2018 | |
|---|
| 1976 | 2019 | /* |
|---|