.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
---|
1 | 2 | /* |
---|
2 | 3 | * INET An implementation of the TCP/IP protocol suite for the LINUX |
---|
3 | 4 | * operating system. INET is implemented using the BSD Socket |
---|
.. | .. |
---|
58 | 59 | * Some other random speedups. |
---|
59 | 60 | * Cyrus Durgin : Cleaned up file for kmod hacks. |
---|
60 | 61 | * Andi Kleen : Fix inet_stream_connect TCP race. |
---|
61 | | - * |
---|
62 | | - * This program is free software; you can redistribute it and/or |
---|
63 | | - * modify it under the terms of the GNU General Public License |
---|
64 | | - * as published by the Free Software Foundation; either version |
---|
65 | | - * 2 of the License, or (at your option) any later version. |
---|
66 | 62 | */ |
---|
67 | 63 | |
---|
68 | 64 | #define pr_fmt(fmt) "IPv4: " fmt |
---|
.. | .. |
---|
120 | 116 | #include <linux/mroute.h> |
---|
121 | 117 | #endif |
---|
122 | 118 | #include <net/l3mdev.h> |
---|
| 119 | +#include <net/compat.h> |
---|
123 | 120 | |
---|
124 | 121 | #include <trace/events/sock.h> |
---|
125 | 122 | |
---|
.. | .. |
---|
136 | 133 | struct inet_sock *inet = inet_sk(sk); |
---|
137 | 134 | |
---|
138 | 135 | __skb_queue_purge(&sk->sk_receive_queue); |
---|
| 136 | + if (sk->sk_rx_skb_cache) { |
---|
| 137 | + __kfree_skb(sk->sk_rx_skb_cache); |
---|
| 138 | + sk->sk_rx_skb_cache = NULL; |
---|
| 139 | + } |
---|
139 | 140 | __skb_queue_purge(&sk->sk_error_queue); |
---|
140 | 141 | |
---|
141 | 142 | sk_mem_reclaim(sk); |
---|
.. | .. |
---|
156 | 157 | WARN_ON(sk->sk_forward_alloc); |
---|
157 | 158 | |
---|
158 | 159 | kfree(rcu_dereference_protected(inet->inet_opt, 1)); |
---|
159 | | - dst_release(rcu_dereference_check(sk->sk_dst_cache, 1)); |
---|
160 | | - dst_release(sk->sk_rx_dst); |
---|
| 160 | + dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1)); |
---|
| 161 | + dst_release(rcu_dereference_protected(sk->sk_rx_dst, 1)); |
---|
161 | 162 | sk_refcnt_debug_dec(sk); |
---|
162 | 163 | } |
---|
163 | 164 | EXPORT_SYMBOL(inet_sock_destruct); |
---|
.. | .. |
---|
208 | 209 | if (!((1 << old_state) & (TCPF_CLOSE | TCPF_LISTEN))) |
---|
209 | 210 | goto out; |
---|
210 | 211 | |
---|
| 212 | + WRITE_ONCE(sk->sk_max_ack_backlog, backlog); |
---|
211 | 213 | /* Really, if the socket is already in listen state |
---|
212 | 214 | * we can only allow the backlog to be adjusted. |
---|
213 | 215 | */ |
---|
.. | .. |
---|
218 | 220 | * because the socket was in TCP_LISTEN state previously but |
---|
219 | 221 | * was shutdown() rather than close(). |
---|
220 | 222 | */ |
---|
221 | | - tcp_fastopen = sock_net(sk)->ipv4.sysctl_tcp_fastopen; |
---|
| 223 | + tcp_fastopen = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen); |
---|
222 | 224 | if ((tcp_fastopen & TFO_SERVER_WO_SOCKOPT1) && |
---|
223 | 225 | (tcp_fastopen & TFO_SERVER_ENABLE) && |
---|
224 | 226 | !inet_csk(sk)->icsk_accept_queue.fastopenq.max_qlen) { |
---|
.. | .. |
---|
231 | 233 | goto out; |
---|
232 | 234 | tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); |
---|
233 | 235 | } |
---|
234 | | - sk->sk_max_ack_backlog = backlog; |
---|
235 | 236 | err = 0; |
---|
236 | 237 | |
---|
237 | 238 | out: |
---|
.. | .. |
---|
337 | 338 | inet->hdrincl = 1; |
---|
338 | 339 | } |
---|
339 | 340 | |
---|
340 | | - if (net->ipv4.sysctl_ip_no_pmtu_disc) |
---|
| 341 | + if (READ_ONCE(net->ipv4.sysctl_ip_no_pmtu_disc)) |
---|
341 | 342 | inet->pmtudisc = IP_PMTUDISC_DONT; |
---|
342 | 343 | else |
---|
343 | 344 | inet->pmtudisc = IP_PMTUDISC_WANT; |
---|
.. | .. |
---|
410 | 411 | if (sk) { |
---|
411 | 412 | long timeout; |
---|
412 | 413 | |
---|
| 414 | + if (!sk->sk_kern_sock) |
---|
| 415 | + BPF_CGROUP_RUN_PROG_INET_SOCK_RELEASE(sk); |
---|
| 416 | + |
---|
413 | 417 | /* Applications forget to leave groups before exiting */ |
---|
414 | 418 | ip_mc_drop_socket(sk); |
---|
415 | 419 | |
---|
.. | .. |
---|
450 | 454 | if (err) |
---|
451 | 455 | return err; |
---|
452 | 456 | |
---|
453 | | - return __inet_bind(sk, uaddr, addr_len, false, true); |
---|
| 457 | + return __inet_bind(sk, uaddr, addr_len, BIND_WITH_LOCK); |
---|
454 | 458 | } |
---|
455 | 459 | EXPORT_SYMBOL(inet_bind); |
---|
456 | 460 | |
---|
457 | 461 | int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, |
---|
458 | | - bool force_bind_address_no_port, bool with_lock) |
---|
| 462 | + u32 flags) |
---|
459 | 463 | { |
---|
460 | 464 | struct sockaddr_in *addr = (struct sockaddr_in *)uaddr; |
---|
461 | 465 | struct inet_sock *inet = inet_sk(sk); |
---|
.. | .. |
---|
494 | 498 | goto out; |
---|
495 | 499 | |
---|
496 | 500 | snum = ntohs(addr->sin_port); |
---|
| 501 | + err = -EPERM; |
---|
| 502 | + if (snum && inet_is_local_unbindable_port(net, snum)) |
---|
| 503 | + goto out; |
---|
| 504 | + |
---|
497 | 505 | err = -EACCES; |
---|
498 | | - if (snum && snum < inet_prot_sock(net) && |
---|
| 506 | + if (snum && inet_port_requires_bind_service(net, snum) && |
---|
499 | 507 | !ns_capable(net->user_ns, CAP_NET_BIND_SERVICE)) |
---|
500 | 508 | goto out; |
---|
501 | 509 | |
---|
.. | .. |
---|
506 | 514 | * would be illegal to use them (multicast/broadcast) in |
---|
507 | 515 | * which case the sending device address is used. |
---|
508 | 516 | */ |
---|
509 | | - if (with_lock) |
---|
| 517 | + if (flags & BIND_WITH_LOCK) |
---|
510 | 518 | lock_sock(sk); |
---|
511 | 519 | |
---|
512 | 520 | /* Check these errors (active socket, double bind). */ |
---|
.. | .. |
---|
520 | 528 | |
---|
521 | 529 | /* Make sure we are allowed to bind here. */ |
---|
522 | 530 | if (snum || !(inet->bind_address_no_port || |
---|
523 | | - force_bind_address_no_port)) { |
---|
| 531 | + (flags & BIND_FORCE_ADDRESS_NO_PORT))) { |
---|
524 | 532 | if (sk->sk_prot->get_port(sk, snum)) { |
---|
525 | 533 | inet->inet_saddr = inet->inet_rcv_saddr = 0; |
---|
526 | 534 | err = -EADDRINUSE; |
---|
527 | 535 | goto out_release_sock; |
---|
528 | 536 | } |
---|
529 | | - err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk); |
---|
530 | | - if (err) { |
---|
531 | | - inet->inet_saddr = inet->inet_rcv_saddr = 0; |
---|
532 | | - goto out_release_sock; |
---|
| 537 | + if (!(flags & BIND_FROM_BPF)) { |
---|
| 538 | + err = BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk); |
---|
| 539 | + if (err) { |
---|
| 540 | + inet->inet_saddr = inet->inet_rcv_saddr = 0; |
---|
| 541 | + goto out_release_sock; |
---|
| 542 | + } |
---|
533 | 543 | } |
---|
534 | 544 | } |
---|
535 | 545 | |
---|
.. | .. |
---|
543 | 553 | sk_dst_reset(sk); |
---|
544 | 554 | err = 0; |
---|
545 | 555 | out_release_sock: |
---|
546 | | - if (with_lock) |
---|
| 556 | + if (flags & BIND_WITH_LOCK) |
---|
547 | 557 | release_sock(sk); |
---|
548 | 558 | out: |
---|
549 | 559 | return err; |
---|
.. | .. |
---|
566 | 576 | return err; |
---|
567 | 577 | } |
---|
568 | 578 | |
---|
569 | | - if (!inet_sk(sk)->inet_num && inet_autobind(sk)) |
---|
| 579 | + if (data_race(!inet_sk(sk)->inet_num) && inet_autobind(sk)) |
---|
570 | 580 | return -EAGAIN; |
---|
571 | 581 | return sk->sk_prot->connect(sk, uaddr, addr_len); |
---|
572 | 582 | } |
---|
.. | .. |
---|
753 | 763 | } |
---|
754 | 764 | EXPORT_SYMBOL(inet_accept); |
---|
755 | 765 | |
---|
756 | | - |
---|
757 | 766 | /* |
---|
758 | 767 | * This does both peername and sockname. |
---|
759 | 768 | */ |
---|
760 | 769 | int inet_getname(struct socket *sock, struct sockaddr *uaddr, |
---|
761 | | - int peer) |
---|
| 770 | + int peer) |
---|
762 | 771 | { |
---|
763 | 772 | struct sock *sk = sock->sk; |
---|
764 | 773 | struct inet_sock *inet = inet_sk(sk); |
---|
.. | .. |
---|
779 | 788 | sin->sin_port = inet->inet_sport; |
---|
780 | 789 | sin->sin_addr.s_addr = addr; |
---|
781 | 790 | } |
---|
| 791 | + if (cgroup_bpf_enabled) |
---|
| 792 | + BPF_CGROUP_RUN_SA_PROG_LOCK(sk, (struct sockaddr *)sin, |
---|
| 793 | + peer ? BPF_CGROUP_INET4_GETPEERNAME : |
---|
| 794 | + BPF_CGROUP_INET4_GETSOCKNAME, |
---|
| 795 | + NULL); |
---|
782 | 796 | memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); |
---|
783 | 797 | return sizeof(*sin); |
---|
784 | 798 | } |
---|
785 | 799 | EXPORT_SYMBOL(inet_getname); |
---|
786 | 800 | |
---|
| 801 | +int inet_send_prepare(struct sock *sk) |
---|
| 802 | +{ |
---|
| 803 | + sock_rps_record_flow(sk); |
---|
| 804 | + |
---|
| 805 | + /* We may need to bind the socket. */ |
---|
| 806 | + if (data_race(!inet_sk(sk)->inet_num) && !sk->sk_prot->no_autobind && |
---|
| 807 | + inet_autobind(sk)) |
---|
| 808 | + return -EAGAIN; |
---|
| 809 | + |
---|
| 810 | + return 0; |
---|
| 811 | +} |
---|
| 812 | +EXPORT_SYMBOL_GPL(inet_send_prepare); |
---|
| 813 | + |
---|
787 | 814 | int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) |
---|
788 | 815 | { |
---|
789 | 816 | struct sock *sk = sock->sk; |
---|
790 | 817 | |
---|
791 | | - sock_rps_record_flow(sk); |
---|
792 | | - |
---|
793 | | - /* We may need to bind the socket. */ |
---|
794 | | - if (!inet_sk(sk)->inet_num && !sk->sk_prot->no_autobind && |
---|
795 | | - inet_autobind(sk)) |
---|
| 818 | + if (unlikely(inet_send_prepare(sk))) |
---|
796 | 819 | return -EAGAIN; |
---|
797 | 820 | |
---|
798 | | - return sk->sk_prot->sendmsg(sk, msg, size); |
---|
| 821 | + return INDIRECT_CALL_2(sk->sk_prot->sendmsg, tcp_sendmsg, udp_sendmsg, |
---|
| 822 | + sk, msg, size); |
---|
799 | 823 | } |
---|
800 | 824 | EXPORT_SYMBOL(inet_sendmsg); |
---|
801 | 825 | |
---|
.. | .. |
---|
804 | 828 | { |
---|
805 | 829 | struct sock *sk = sock->sk; |
---|
806 | 830 | |
---|
807 | | - sock_rps_record_flow(sk); |
---|
808 | | - |
---|
809 | | - /* We may need to bind the socket. */ |
---|
810 | | - if (!inet_sk(sk)->inet_num && !sk->sk_prot->no_autobind && |
---|
811 | | - inet_autobind(sk)) |
---|
| 831 | + if (unlikely(inet_send_prepare(sk))) |
---|
812 | 832 | return -EAGAIN; |
---|
813 | 833 | |
---|
814 | 834 | if (sk->sk_prot->sendpage) |
---|
.. | .. |
---|
817 | 837 | } |
---|
818 | 838 | EXPORT_SYMBOL(inet_sendpage); |
---|
819 | 839 | |
---|
| 840 | +INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *, struct msghdr *, |
---|
| 841 | + size_t, int, int, int *)); |
---|
820 | 842 | int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, |
---|
821 | 843 | int flags) |
---|
822 | 844 | { |
---|
.. | .. |
---|
827 | 849 | if (likely(!(flags & MSG_ERRQUEUE))) |
---|
828 | 850 | sock_rps_record_flow(sk); |
---|
829 | 851 | |
---|
830 | | - err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, |
---|
831 | | - flags & ~MSG_DONTWAIT, &addr_len); |
---|
| 852 | + err = INDIRECT_CALL_2(sk->sk_prot->recvmsg, tcp_recvmsg, udp_recvmsg, |
---|
| 853 | + sk, msg, size, flags & MSG_DONTWAIT, |
---|
| 854 | + flags & ~MSG_DONTWAIT, &addr_len); |
---|
832 | 855 | if (err >= 0) |
---|
833 | 856 | msg->msg_namelen = addr_len; |
---|
834 | 857 | return err; |
---|
.. | .. |
---|
863 | 886 | err = -ENOTCONN; |
---|
864 | 887 | /* Hack to wake up other listeners, who can poll for |
---|
865 | 888 | EPOLLHUP, even on eg. unconnected UDP sockets -- RR */ |
---|
866 | | - /* fall through */ |
---|
| 889 | + fallthrough; |
---|
867 | 890 | default: |
---|
868 | | - sk->sk_shutdown |= how; |
---|
| 891 | + WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | how); |
---|
869 | 892 | if (sk->sk_prot->shutdown) |
---|
870 | 893 | sk->sk_prot->shutdown(sk, how); |
---|
871 | 894 | break; |
---|
.. | .. |
---|
877 | 900 | case TCP_LISTEN: |
---|
878 | 901 | if (!(how & RCV_SHUTDOWN)) |
---|
879 | 902 | break; |
---|
880 | | - /* fall through */ |
---|
| 903 | + fallthrough; |
---|
881 | 904 | case TCP_SYN_SENT: |
---|
882 | 905 | err = sk->sk_prot->disconnect(sk, O_NONBLOCK); |
---|
883 | 906 | sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED; |
---|
.. | .. |
---|
911 | 934 | struct rtentry rt; |
---|
912 | 935 | |
---|
913 | 936 | switch (cmd) { |
---|
914 | | - case SIOCGSTAMP: |
---|
915 | | - err = sock_get_timestamp(sk, (struct timeval __user *)arg); |
---|
916 | | - break; |
---|
917 | | - case SIOCGSTAMPNS: |
---|
918 | | - err = sock_get_timestampns(sk, (struct timespec __user *)arg); |
---|
919 | | - break; |
---|
920 | 937 | case SIOCADDRT: |
---|
921 | 938 | case SIOCDELRT: |
---|
922 | 939 | if (copy_from_user(&rt, p, sizeof(struct rtentry))) |
---|
.. | .. |
---|
965 | 982 | EXPORT_SYMBOL(inet_ioctl); |
---|
966 | 983 | |
---|
967 | 984 | #ifdef CONFIG_COMPAT |
---|
| 985 | +static int inet_compat_routing_ioctl(struct sock *sk, unsigned int cmd, |
---|
| 986 | + struct compat_rtentry __user *ur) |
---|
| 987 | +{ |
---|
| 988 | + compat_uptr_t rtdev; |
---|
| 989 | + struct rtentry rt; |
---|
| 990 | + |
---|
| 991 | + if (copy_from_user(&rt.rt_dst, &ur->rt_dst, |
---|
| 992 | + 3 * sizeof(struct sockaddr)) || |
---|
| 993 | + get_user(rt.rt_flags, &ur->rt_flags) || |
---|
| 994 | + get_user(rt.rt_metric, &ur->rt_metric) || |
---|
| 995 | + get_user(rt.rt_mtu, &ur->rt_mtu) || |
---|
| 996 | + get_user(rt.rt_window, &ur->rt_window) || |
---|
| 997 | + get_user(rt.rt_irtt, &ur->rt_irtt) || |
---|
| 998 | + get_user(rtdev, &ur->rt_dev)) |
---|
| 999 | + return -EFAULT; |
---|
| 1000 | + |
---|
| 1001 | + rt.rt_dev = compat_ptr(rtdev); |
---|
| 1002 | + return ip_rt_ioctl(sock_net(sk), cmd, &rt); |
---|
| 1003 | +} |
---|
| 1004 | + |
---|
968 | 1005 | static int inet_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) |
---|
969 | 1006 | { |
---|
| 1007 | + void __user *argp = compat_ptr(arg); |
---|
970 | 1008 | struct sock *sk = sock->sk; |
---|
971 | | - int err = -ENOIOCTLCMD; |
---|
972 | 1009 | |
---|
973 | | - if (sk->sk_prot->compat_ioctl) |
---|
974 | | - err = sk->sk_prot->compat_ioctl(sk, cmd, arg); |
---|
975 | | - |
---|
976 | | - return err; |
---|
| 1010 | + switch (cmd) { |
---|
| 1011 | + case SIOCADDRT: |
---|
| 1012 | + case SIOCDELRT: |
---|
| 1013 | + return inet_compat_routing_ioctl(sk, cmd, argp); |
---|
| 1014 | + default: |
---|
| 1015 | + if (!sk->sk_prot->compat_ioctl) |
---|
| 1016 | + return -ENOIOCTLCMD; |
---|
| 1017 | + return sk->sk_prot->compat_ioctl(sk, cmd, arg); |
---|
| 1018 | + } |
---|
977 | 1019 | } |
---|
978 | | -#endif |
---|
| 1020 | +#endif /* CONFIG_COMPAT */ |
---|
979 | 1021 | |
---|
980 | 1022 | const struct proto_ops inet_stream_ops = { |
---|
981 | 1023 | .family = PF_INET, |
---|
.. | .. |
---|
988 | 1030 | .getname = inet_getname, |
---|
989 | 1031 | .poll = tcp_poll, |
---|
990 | 1032 | .ioctl = inet_ioctl, |
---|
| 1033 | + .gettstamp = sock_gettstamp, |
---|
991 | 1034 | .listen = inet_listen, |
---|
992 | 1035 | .shutdown = inet_shutdown, |
---|
993 | 1036 | .setsockopt = sock_common_setsockopt, |
---|
.. | .. |
---|
1004 | 1047 | .sendpage_locked = tcp_sendpage_locked, |
---|
1005 | 1048 | .peek_len = tcp_peek_len, |
---|
1006 | 1049 | #ifdef CONFIG_COMPAT |
---|
1007 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
---|
1008 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
---|
1009 | 1050 | .compat_ioctl = inet_compat_ioctl, |
---|
1010 | 1051 | #endif |
---|
1011 | 1052 | .set_rcvlowat = tcp_set_rcvlowat, |
---|
.. | .. |
---|
1023 | 1064 | .getname = inet_getname, |
---|
1024 | 1065 | .poll = udp_poll, |
---|
1025 | 1066 | .ioctl = inet_ioctl, |
---|
| 1067 | + .gettstamp = sock_gettstamp, |
---|
1026 | 1068 | .listen = sock_no_listen, |
---|
1027 | 1069 | .shutdown = inet_shutdown, |
---|
1028 | 1070 | .setsockopt = sock_common_setsockopt, |
---|
.. | .. |
---|
1033 | 1075 | .sendpage = inet_sendpage, |
---|
1034 | 1076 | .set_peek_off = sk_set_peek_off, |
---|
1035 | 1077 | #ifdef CONFIG_COMPAT |
---|
1036 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
---|
1037 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
---|
1038 | 1078 | .compat_ioctl = inet_compat_ioctl, |
---|
1039 | 1079 | #endif |
---|
1040 | 1080 | }; |
---|
.. | .. |
---|
1055 | 1095 | .getname = inet_getname, |
---|
1056 | 1096 | .poll = datagram_poll, |
---|
1057 | 1097 | .ioctl = inet_ioctl, |
---|
| 1098 | + .gettstamp = sock_gettstamp, |
---|
1058 | 1099 | .listen = sock_no_listen, |
---|
1059 | 1100 | .shutdown = inet_shutdown, |
---|
1060 | 1101 | .setsockopt = sock_common_setsockopt, |
---|
.. | .. |
---|
1064 | 1105 | .mmap = sock_no_mmap, |
---|
1065 | 1106 | .sendpage = inet_sendpage, |
---|
1066 | 1107 | #ifdef CONFIG_COMPAT |
---|
1067 | | - .compat_setsockopt = compat_sock_common_setsockopt, |
---|
1068 | | - .compat_getsockopt = compat_sock_common_getsockopt, |
---|
1069 | 1108 | .compat_ioctl = inet_compat_ioctl, |
---|
1070 | 1109 | #endif |
---|
1071 | 1110 | }; |
---|
.. | .. |
---|
1209 | 1248 | if (new_saddr == old_saddr) |
---|
1210 | 1249 | return 0; |
---|
1211 | 1250 | |
---|
1212 | | - if (sock_net(sk)->ipv4.sysctl_ip_dynaddr > 1) { |
---|
| 1251 | + if (READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) > 1) { |
---|
1213 | 1252 | pr_info("%s(): shifting inet->saddr from %pI4 to %pI4\n", |
---|
1214 | 1253 | __func__, &old_saddr, &new_saddr); |
---|
1215 | 1254 | } |
---|
.. | .. |
---|
1264 | 1303 | * Other protocols have to map its equivalent state to TCP_SYN_SENT. |
---|
1265 | 1304 | * DCCP maps its DCCP_REQUESTING state to TCP_SYN_SENT. -acme |
---|
1266 | 1305 | */ |
---|
1267 | | - if (!sock_net(sk)->ipv4.sysctl_ip_dynaddr || |
---|
| 1306 | + if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) || |
---|
1268 | 1307 | sk->sk_state != TCP_SYN_SENT || |
---|
1269 | 1308 | (sk->sk_userlocks & SOCK_BINDADDR_LOCK) || |
---|
1270 | 1309 | (err = inet_sk_reselect_saddr(sk)) != 0) |
---|
.. | .. |
---|
1388 | 1427 | } |
---|
1389 | 1428 | EXPORT_SYMBOL(inet_gso_segment); |
---|
1390 | 1429 | |
---|
| 1430 | +static struct sk_buff *ipip_gso_segment(struct sk_buff *skb, |
---|
| 1431 | + netdev_features_t features) |
---|
| 1432 | +{ |
---|
| 1433 | + if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4)) |
---|
| 1434 | + return ERR_PTR(-EINVAL); |
---|
| 1435 | + |
---|
| 1436 | + return inet_gso_segment(skb, features); |
---|
| 1437 | +} |
---|
| 1438 | + |
---|
1391 | 1439 | struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) |
---|
1392 | 1440 | { |
---|
1393 | 1441 | const struct net_offload *ops; |
---|
.. | .. |
---|
1497 | 1545 | skb_gro_pull(skb, sizeof(*iph)); |
---|
1498 | 1546 | skb_set_transport_header(skb, skb_gro_offset(skb)); |
---|
1499 | 1547 | |
---|
1500 | | - pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); |
---|
| 1548 | + pp = indirect_call_gro_receive(tcp4_gro_receive, udp4_gro_receive, |
---|
| 1549 | + ops->callbacks.gro_receive, head, skb); |
---|
1501 | 1550 | |
---|
1502 | 1551 | out_unlock: |
---|
1503 | 1552 | rcu_read_unlock(); |
---|
.. | .. |
---|
1584 | 1633 | * because any hdr with option will have been flushed in |
---|
1585 | 1634 | * inet_gro_receive(). |
---|
1586 | 1635 | */ |
---|
1587 | | - err = ops->callbacks.gro_complete(skb, nhoff + sizeof(*iph)); |
---|
| 1636 | + err = INDIRECT_CALL_2(ops->callbacks.gro_complete, |
---|
| 1637 | + tcp4_gro_complete, udp4_gro_complete, |
---|
| 1638 | + skb, nhoff + sizeof(*iph)); |
---|
1588 | 1639 | |
---|
1589 | 1640 | out_unlock: |
---|
1590 | 1641 | rcu_read_unlock(); |
---|
.. | .. |
---|
1678 | 1729 | }; |
---|
1679 | 1730 | #endif |
---|
1680 | 1731 | |
---|
1681 | | -/* thinking of making this const? Don't. |
---|
1682 | | - * early_demux can change based on sysctl. |
---|
1683 | | - */ |
---|
1684 | | -static struct net_protocol tcp_protocol = { |
---|
1685 | | - .early_demux = tcp_v4_early_demux, |
---|
1686 | | - .early_demux_handler = tcp_v4_early_demux, |
---|
| 1732 | +static const struct net_protocol tcp_protocol = { |
---|
1687 | 1733 | .handler = tcp_v4_rcv, |
---|
1688 | 1734 | .err_handler = tcp_v4_err, |
---|
1689 | 1735 | .no_policy = 1, |
---|
.. | .. |
---|
1691 | 1737 | .icmp_strict_tag_validation = 1, |
---|
1692 | 1738 | }; |
---|
1693 | 1739 | |
---|
1694 | | -/* thinking of making this const? Don't. |
---|
1695 | | - * early_demux can change based on sysctl. |
---|
1696 | | - */ |
---|
1697 | | -static struct net_protocol udp_protocol = { |
---|
1698 | | - .early_demux = udp_v4_early_demux, |
---|
1699 | | - .early_demux_handler = udp_v4_early_demux, |
---|
| 1740 | +static const struct net_protocol udp_protocol = { |
---|
1700 | 1741 | .handler = udp_rcv, |
---|
1701 | 1742 | .err_handler = udp_err, |
---|
1702 | 1743 | .no_policy = 1, |
---|
.. | .. |
---|
1772 | 1813 | free_percpu(net->mib.net_statistics); |
---|
1773 | 1814 | free_percpu(net->mib.ip_statistics); |
---|
1774 | 1815 | free_percpu(net->mib.tcp_statistics); |
---|
| 1816 | +#ifdef CONFIG_MPTCP |
---|
| 1817 | + /* allocated on demand, see mptcp_init_sock() */ |
---|
| 1818 | + free_percpu(net->mib.mptcp_statistics); |
---|
| 1819 | +#endif |
---|
1775 | 1820 | } |
---|
1776 | 1821 | |
---|
1777 | 1822 | static __net_initdata struct pernet_operations ipv4_mib_ops = { |
---|
.. | .. |
---|
1810 | 1855 | net->ipv4.sysctl_ip_early_demux = 1; |
---|
1811 | 1856 | net->ipv4.sysctl_udp_early_demux = 1; |
---|
1812 | 1857 | net->ipv4.sysctl_tcp_early_demux = 1; |
---|
| 1858 | + net->ipv4.sysctl_nexthop_compat_mode = 1; |
---|
1813 | 1859 | #ifdef CONFIG_SYSCTL |
---|
1814 | 1860 | net->ipv4.sysctl_ip_prot_sock = PROT_SOCK; |
---|
1815 | 1861 | #endif |
---|
.. | .. |
---|
1824 | 1870 | return 0; |
---|
1825 | 1871 | } |
---|
1826 | 1872 | |
---|
1827 | | -static __net_exit void inet_exit_net(struct net *net) |
---|
1828 | | -{ |
---|
1829 | | -} |
---|
1830 | | - |
---|
1831 | 1873 | static __net_initdata struct pernet_operations af_inet_ops = { |
---|
1832 | 1874 | .init = inet_init_net, |
---|
1833 | | - .exit = inet_exit_net, |
---|
1834 | 1875 | }; |
---|
1835 | 1876 | |
---|
1836 | 1877 | static int __init init_inet_pernet_ops(void) |
---|
.. | .. |
---|
1855 | 1896 | |
---|
1856 | 1897 | static const struct net_offload ipip_offload = { |
---|
1857 | 1898 | .callbacks = { |
---|
1858 | | - .gso_segment = inet_gso_segment, |
---|
| 1899 | + .gso_segment = ipip_gso_segment, |
---|
1859 | 1900 | .gro_receive = ipip_gro_receive, |
---|
1860 | 1901 | .gro_complete = ipip_gro_complete, |
---|
1861 | 1902 | }, |
---|
.. | .. |
---|
1894 | 1935 | { |
---|
1895 | 1936 | struct inet_protosw *q; |
---|
1896 | 1937 | struct list_head *r; |
---|
1897 | | - int rc = -EINVAL; |
---|
| 1938 | + int rc; |
---|
1898 | 1939 | |
---|
1899 | 1940 | sock_skb_cb_check_size(sizeof(struct inet_skb_parm)); |
---|
1900 | 1941 | |
---|
.. | .. |
---|
1971 | 2012 | /* Add UDP-Lite (RFC 3828) */ |
---|
1972 | 2013 | udplite4_register(); |
---|
1973 | 2014 | |
---|
| 2015 | + raw_init(); |
---|
| 2016 | + |
---|
1974 | 2017 | ping_init(); |
---|
1975 | 2018 | |
---|
1976 | 2019 | /* |
---|