forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/include/net/sock.h
....@@ -1,3 +1,4 @@
1
+/* SPDX-License-Identifier: GPL-2.0-or-later */
12 /*
23  * INET		An implementation of the TCP/IP protocol suite for the LINUX
34  *		operating system.  INET is implemented using the  BSD Socket
....@@ -30,12 +31,6 @@
3031  *              			respective headers and ipv4/v6, etc now
3132  *              			use private slabcaches for its socks
3233  *              Pedro Hortas	:	New flags field for socket options
33
- *
34
- *
35
- *		This program is free software; you can redistribute it and/or
36
- *		modify it under the terms of the GNU General Public License
37
- *		as published by the Free Software Foundation; either version
38
- *		2 of the License, or (at your option) any later version.
3934  */
4035 #ifndef _SOCK_H
4136 #define _SOCK_H
....@@ -64,6 +59,7 @@
6459 #include <linux/filter.h>
6560 #include <linux/rculist_nulls.h>
6661 #include <linux/poll.h>
62
+#include <linux/sockptr.h>
6763 
6864 #include <linux/atomic.h>
6965 #include <linux/refcount.h>
....@@ -71,9 +67,9 @@
7167 #include <net/checksum.h>
7268 #include <net/tcp_states.h>
7369 #include <linux/net_tstamp.h>
74
-#include <net/smc.h>
7570 #include <net/l3mdev.h>
7671 #include <linux/android_kabi.h>
72
+#include <linux/android_vendor.h>
7773 
7874 /*
7975  * This structure really needs to be cleaned up.
....@@ -124,19 +120,26 @@
124120  *	struct sock_common - minimal network layer representation of sockets
125121  *	@skc_daddr: Foreign IPv4 addr
126122  *	@skc_rcv_saddr: Bound local IPv4 addr
123
+ *	@skc_addrpair: 8-byte-aligned __u64 union of @skc_daddr & @skc_rcv_saddr
127124  *	@skc_hash: hash value used with various protocol lookup tables
128125  *	@skc_u16hashes: two u16 hash values used by UDP lookup tables
129126  *	@skc_dport: placeholder for inet_dport/tw_dport
130127  *	@skc_num: placeholder for inet_num/tw_num
128
+ *	@skc_portpair: __u32 union of @skc_dport & @skc_num
131129  *	@skc_family: network address family
132130  *	@skc_state: Connection state
133131  *	@skc_reuse: %SO_REUSEADDR setting
134132  *	@skc_reuseport: %SO_REUSEPORT setting
133
+ *	@skc_ipv6only: socket is IPV6 only
134
+ *	@skc_net_refcnt: socket is using net ref counting
135135  *	@skc_bound_dev_if: bound device index if != 0
136136  *	@skc_bind_node: bind hash linkage for various protocol lookup tables
137137  *	@skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
138138  *	@skc_prot: protocol handlers inside a network family
139139  *	@skc_net: reference to the network namespace of this socket
140
+ *	@skc_v6_daddr: IPV6 destination address
141
+ *	@skc_v6_rcv_saddr: IPV6 source address
142
+ *	@skc_cookie: socket's cookie value
140143  *	@skc_node: main hash linkage for various protocol lookup tables
141144  *	@skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
142145  *	@skc_tx_queue_mapping: tx queue number for this connection
....@@ -144,16 +147,21 @@
144147  *	@skc_flags: place holder for sk_flags
145148  *		%SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
146149  *		%SO_OOBINLINE settings, %SO_TIMESTAMPING settings
150
+ *	@skc_listener: connection request listener socket (aka rsk_listener)
151
+ *		[union with @skc_flags]
152
+ *	@skc_tw_dr: (aka tw_dr) ptr to &struct inet_timewait_death_row
153
+ *		[union with @skc_flags]
147154  *	@skc_incoming_cpu: record/match cpu processing incoming packets
155
+ *	@skc_rcv_wnd: (aka rsk_rcv_wnd) TCP receive window size (possibly scaled)
156
+ *		[union with @skc_incoming_cpu]
157
+ *	@skc_tw_rcv_nxt: (aka tw_rcv_nxt) TCP window next expected seq number
158
+ *		[union with @skc_incoming_cpu]
148159  *	@skc_refcnt: reference count
149160  *
150161  *	This is the minimal network layer representation of sockets, the header
151162  *	for struct sock and struct inet_timewait_sock.
152163  */
153164 struct sock_common {
154
-	/* skc_daddr and skc_rcv_saddr must be grouped on a 8 bytes aligned
155
-	 * address on 64bit arches : cf INET_MATCH()
156
-	 */
157165 	union {
158166 		__addrpair	skc_addrpair;
159167 		struct {
....@@ -237,6 +245,8 @@
237245 	/* public: */
238246 };
239247 
248
+struct bpf_local_storage;
249
+
240250 /**
241251   *	struct sock - network layer representation of sockets
242252   *	@__sk_common: shared layout with inet_timewait_sock
....@@ -250,6 +260,7 @@
250260   *	@sk_dst_cache: destination cache
251261   *	@sk_dst_pending_confirm: need to confirm neighbour
252262   *	@sk_policy: flow policy
263
+  *	@sk_rx_skb_cache: cache copy of recently accessed RX skb
253264   *	@sk_receive_queue: incoming packets
254265   *	@sk_wmem_alloc: transmit queue bytes committed
255266   *	@sk_tsq_flags: TCP Small Queues flags
....@@ -270,6 +281,8 @@
270281   *	@sk_no_check_rx: allow zero checksum in RX packets
271282   *	@sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
272283   *	@sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK)
284
+  *	@sk_route_forced_caps: static, forced route capabilities
285
+  *		(set in tcp_init_sock())
273286   *	@sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
274287   *	@sk_gso_max_size: Maximum GSO segment size to build
275288   *	@sk_gso_max_segs: Maximum number of GSO segments
....@@ -308,6 +321,8 @@
308321   *	@sk_frag: cached page frag
309322   *	@sk_peek_off: current peek_offset value
310323   *	@sk_send_head: front of stuff to transmit
324
+  *	@tcp_rtx_queue: TCP re-transmit queue [union with @sk_send_head]
325
+  *	@sk_tx_skb_cache: cache copy of recently accessed TX skb
311326   *	@sk_security: used by security modules
312327   *	@sk_mark: generic packet mark
313328   *	@sk_cgrp_data: cgroup data for this cgroup
....@@ -318,11 +333,14 @@
318333   *	@sk_write_space: callback to indicate there is bf sending space available
319334   *	@sk_error_report: callback to indicate errors (e.g. %MSG_ERRQUEUE)
320335   *	@sk_backlog_rcv: callback to process the backlog
336
+  *	@sk_validate_xmit_skb: ptr to an optional validate function
321337   *	@sk_destruct: called at sock freeing time, i.e. when all refcnt == 0
322338   *	@sk_reuseport_cb: reuseport group container
339
+  *	@sk_bpf_storage: ptr to cache and control for bpf_sk_storage
323340   *	@sk_rcu: used during RCU grace period
324341   *	@sk_clockid: clockid used by time-based scheduling (SO_TXTIME)
325342   *	@sk_txtime_deadline_mode: set deadline mode for SO_TXTIME
343
+  *	@sk_txtime_report_errors: set report errors mode for SO_TXTIME
326344   *	@sk_txtime_unused: unused txtime flags
327345   */
328346 struct sock {
....@@ -369,6 +387,7 @@
369387 	atomic_t		sk_drops;
370388 	int			sk_rcvlowat;
371389 	struct sk_buff_head	sk_error_queue;
390
+	struct sk_buff		*sk_rx_skb_cache;
372391 	struct sk_buff_head	sk_receive_queue;
373392 	/*
374393 	 * The backlog queue is special, it is always used with
....@@ -397,12 +416,14 @@
397416 	struct sk_filter __rcu	*sk_filter;
398417 	union {
399418 		struct socket_wq __rcu	*sk_wq;
419
+		/* private: */
400420 		struct socket_wq	*sk_wq_raw;
421
+		/* public: */
401422 	};
402423 #ifdef CONFIG_XFRM
403424 	struct xfrm_policy __rcu *sk_policy[2];
404425 #endif
405
-	struct dst_entry	*sk_rx_dst;
426
+	struct dst_entry __rcu	*sk_rx_dst;
406427 	struct dst_entry __rcu	*sk_dst_cache;
407428 	atomic_t		sk_omem_alloc;
408429 	int			sk_sndbuf;
....@@ -415,6 +436,7 @@
415436 		struct sk_buff	*sk_send_head;
416437 		struct rb_root	tcp_rtx_queue;
417438 	};
439
+	struct sk_buff		*sk_tx_skb_cache;
418440 	struct sk_buff_head	sk_write_queue;
419441 	__s32			sk_peek_off;
420442 	int			sk_write_pending;
....@@ -424,8 +446,8 @@
424446 	struct timer_list	sk_timer;
425447 	__u32			sk_priority;
426448 	__u32			sk_mark;
427
-	u32			sk_pacing_rate; /* bytes per second */
428
-	u32			sk_max_pacing_rate;
449
+	unsigned long		sk_pacing_rate; /* bytes per second */
450
+	unsigned long		sk_max_pacing_rate;
429451 	struct page_frag	sk_frag;
430452 	netdev_features_t	sk_route_caps;
431453 	netdev_features_t	sk_route_nocaps;
....@@ -439,31 +461,15 @@
439461 	 * Because of non atomicity rules, all
440462 	 * changes are protected by socket lock.
441463 	 */
442
-	unsigned int		__sk_flags_offset[0];
443
-#ifdef __BIG_ENDIAN_BITFIELD
444
-#define SK_FL_PROTO_SHIFT  16
445
-#define SK_FL_PROTO_MASK   0x00ff0000
446
-
447
-#define SK_FL_TYPE_SHIFT   0
448
-#define SK_FL_TYPE_MASK    0x0000ffff
449
-#else
450
-#define SK_FL_PROTO_SHIFT  8
451
-#define SK_FL_PROTO_MASK   0x0000ff00
452
-
453
-#define SK_FL_TYPE_SHIFT   16
454
-#define SK_FL_TYPE_MASK    0xffff0000
455
-#endif
456
-
457
-	unsigned int		sk_padding : 1,
464
+	u8			sk_padding : 1,
458465 				sk_kern_sock : 1,
459466 				sk_no_check_tx : 1,
460467 				sk_no_check_rx : 1,
461
-				sk_userlocks : 4,
462
-				sk_protocol  : 8,
463
-				sk_type      : 16;
464
-#define SK_PROTOCOL_MAX U8_MAX
465
-	u16			sk_gso_max_segs;
468
+				sk_userlocks : 4;
466469 	u8			sk_pacing_shift;
470
+	u16			sk_type;
471
+	u16			sk_protocol;
472
+	u16			sk_gso_max_segs;
467473 	unsigned long	        sk_lingertime;
468474 	struct proto		*sk_prot_creator;
469475 	rwlock_t		sk_callback_lock;
....@@ -472,7 +478,7 @@
472478 	u32			sk_ack_backlog;
473479 	u32			sk_max_ack_backlog;
474480 	kuid_t			sk_uid;
475
-#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC) || IS_ENABLED(CONFIG_PREEMPT_RT)
481
+#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)
476482 	spinlock_t		sk_peer_lock;
477483 #else
478484 	/* sk_peer_lock is in the ANDROID_KABI_RESERVE(1) field below */
....@@ -515,9 +521,12 @@
515521 #endif
516522 	void                    (*sk_destruct)(struct sock *sk);
517523 	struct sock_reuseport __rcu	*sk_reuseport_cb;
524
+#ifdef CONFIG_BPF_SYSCALL
525
+	struct bpf_local_storage __rcu	*sk_bpf_storage;
526
+#endif
518527 	struct rcu_head		sk_rcu;
519528 
520
-#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC) || IS_ENABLED(CONFIG_PREEMPT_RT)
529
+#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK) || IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)
521530 	ANDROID_KABI_RESERVE(1);
522531 #else
523532 	ANDROID_KABI_USE(1, spinlock_t sk_peer_lock);
....@@ -529,6 +538,8 @@
529538 	ANDROID_KABI_RESERVE(6);
530539 	ANDROID_KABI_RESERVE(7);
531540 	ANDROID_KABI_RESERVE(8);
541
+
542
+	ANDROID_OEM_DATA(1);
532543 };
533544 
534545 enum sk_pacing {
....@@ -537,10 +548,72 @@
537548 	SK_PACING_FQ		= 2,
538549 };
539550 
551
+/* flag bits in sk_user_data
552
+ *
553
+ * - SK_USER_DATA_NOCOPY:      Pointer stored in sk_user_data might
554
+ *   not be suitable for copying when cloning the socket. For instance,
555
+ *   it can point to a reference counted object. sk_user_data bottom
556
+ *   bit is set if pointer must not be copied.
557
+ *
558
+ * - SK_USER_DATA_BPF:         Mark whether sk_user_data field is
559
+ *   managed/owned by a BPF reuseport array. This bit should be set
560
+ *   when sk_user_data's sk is added to the bpf's reuseport_array.
561
+ *
562
+ * - SK_USER_DATA_PSOCK:       Mark whether pointer stored in
563
+ *   sk_user_data points to psock type. This bit should be set
564
+ *   when sk_user_data is assigned to a psock object.
565
+ */
566
+#define SK_USER_DATA_NOCOPY	1UL
567
+#define SK_USER_DATA_BPF	2UL
568
+#define SK_USER_DATA_PSOCK	4UL
569
+#define SK_USER_DATA_PTRMASK	~(SK_USER_DATA_NOCOPY | SK_USER_DATA_BPF |\
570
+				  SK_USER_DATA_PSOCK)
571
+
572
+/**
573
+ * sk_user_data_is_nocopy - Test if sk_user_data pointer must not be copied
574
+ * @sk: socket
575
+ */
576
+static inline bool sk_user_data_is_nocopy(const struct sock *sk)
577
+{
578
+	return ((uintptr_t)sk->sk_user_data & SK_USER_DATA_NOCOPY);
579
+}
580
+
540581 #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
541582 
542
-#define rcu_dereference_sk_user_data(sk)	rcu_dereference(__sk_user_data((sk)))
543
-#define rcu_assign_sk_user_data(sk, ptr)	rcu_assign_pointer(__sk_user_data((sk)), ptr)
583
+/**
584
+ * __rcu_dereference_sk_user_data_with_flags - return the pointer
585
+ * only if argument flags all has been set in sk_user_data. Otherwise
586
+ * return NULL
587
+ *
588
+ * @sk: socket
589
+ * @flags: flag bits
590
+ */
591
+static inline void *
592
+__rcu_dereference_sk_user_data_with_flags(const struct sock *sk,
593
+					  uintptr_t flags)
594
+{
595
+	uintptr_t sk_user_data = (uintptr_t)rcu_dereference(__sk_user_data(sk));
596
+
597
+	WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK);
598
+
599
+	if ((sk_user_data & flags) == flags)
600
+		return (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
601
+	return NULL;
602
+}
603
+
604
+#define rcu_dereference_sk_user_data(sk)				\
605
+	__rcu_dereference_sk_user_data_with_flags(sk, 0)
606
+#define __rcu_assign_sk_user_data_with_flags(sk, ptr, flags)		\
607
+({									\
608
+	uintptr_t __tmp1 = (uintptr_t)(ptr),				\
609
+		  __tmp2 = (uintptr_t)(flags);				\
610
+	WARN_ON_ONCE(__tmp1 & ~SK_USER_DATA_PTRMASK);			\
611
+	WARN_ON_ONCE(__tmp2 & SK_USER_DATA_PTRMASK);			\
612
+	rcu_assign_pointer(__sk_user_data((sk)),			\
613
+			   __tmp1 | __tmp2);				\
614
+})
615
+#define rcu_assign_sk_user_data(sk, ptr)				\
616
+	__rcu_assign_sk_user_data_with_flags(sk, ptr, 0)
544617 
545618 /*
546619  * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK
....@@ -820,7 +893,6 @@
820893 	SOCK_RCVTSTAMP, /* %SO_TIMESTAMP setting */
821894 	SOCK_RCVTSTAMPNS, /* %SO_TIMESTAMPNS setting */
822895 	SOCK_LOCALROUTE, /* route locally only, %SO_DONTROUTE setting */
823
-	SOCK_QUEUE_SHRUNK, /* write queue has been shrunk recently */
824896 	SOCK_MEMALLOC, /* VM depends on this socket for swapping */
825897 	SOCK_TIMESTAMPING_RX_SOFTWARE,  /* %SOF_TIMESTAMPING_RX_SOFTWARE */
826898 	SOCK_FASYNC, /* fasync() active */
....@@ -835,6 +907,8 @@
835907 	SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
836908 	SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
837909 	SOCK_TXTIME,
910
+	SOCK_XDP, /* XDP is attached */
911
+	SOCK_TSTAMP_NEW, /* Indicates 64 bit timestamps always */
838912 };
839913 
840914 #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
....@@ -852,6 +926,15 @@
852926 static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
853927 {
854928 	__clear_bit(flag, &sk->sk_flags);
929
+}
930
+
931
+static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
932
+				     int valbool)
933
+{
934
+	if (valbool)
935
+		sock_set_flag(sk, bit);
936
+	else
937
+		sock_reset_flag(sk, bit);
855938 }
856939 
857940 static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
....@@ -885,17 +968,17 @@
885968 
886969 static inline void sk_acceptq_removed(struct sock *sk)
887970 {
888
-	sk->sk_ack_backlog--;
971
+	WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog - 1);
889972 }
890973 
891974 static inline void sk_acceptq_added(struct sock *sk)
892975 {
893
-	sk->sk_ack_backlog++;
976
+	WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog + 1);
894977 }
895978 
896979 static inline bool sk_acceptq_is_full(const struct sock *sk)
897980 {
898
-	return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
981
+	return READ_ONCE(sk->sk_ack_backlog) > READ_ONCE(sk->sk_max_ack_backlog);
899982 }
900983 
901984 /*
....@@ -903,12 +986,17 @@
903986  */
904987 static inline int sk_stream_min_wspace(const struct sock *sk)
905988 {
906
-	return sk->sk_wmem_queued >> 1;
989
+	return READ_ONCE(sk->sk_wmem_queued) >> 1;
907990 }
908991 
909992 static inline int sk_stream_wspace(const struct sock *sk)
910993 {
911
-	return sk->sk_sndbuf - sk->sk_wmem_queued;
994
+	return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
995
+}
996
+
997
+static inline void sk_wmem_queued_add(struct sock *sk, int val)
998
+{
999
+	WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
9121000 }
9131001 
9141002 void sk_stream_write_space(struct sock *sk);
....@@ -993,7 +1081,7 @@
9931081 static inline void sock_rps_record_flow(const struct sock *sk)
9941082 {
9951083 #ifdef CONFIG_RPS
996
-	if (static_key_false(&rfs_needed)) {
1084
+	if (static_branch_unlikely(&rfs_needed)) {
9971085 		/* Reading sk->sk_rxhash might incur an expensive cache line
9981086 		 * miss.
9991087 		 *
....@@ -1104,21 +1192,13 @@
11041192 	void			(*destroy)(struct sock *sk);
11051193 	void			(*shutdown)(struct sock *sk, int how);
11061194 	int			(*setsockopt)(struct sock *sk, int level,
1107
-					int optname, char __user *optval,
1195
+					int optname, sockptr_t optval,
11081196 					unsigned int optlen);
11091197 	int			(*getsockopt)(struct sock *sk, int level,
11101198 					int optname, char __user *optval,
11111199 					int __user *option);
11121200 	void			(*keepalive)(struct sock *sk, int valbool);
11131201 #ifdef CONFIG_COMPAT
1114
-	int			(*compat_setsockopt)(struct sock *sk,
1115
-					int level,
1116
-					int optname, char __user *optval,
1117
-					unsigned int optlen);
1118
-	int			(*compat_getsockopt)(struct sock *sk,
1119
-					int level,
1120
-					int optname, char __user *optval,
1121
-					int __user *option);
11221202 	int			(*compat_ioctl)(struct sock *sk,
11231203 					unsigned int cmd, unsigned long arg);
11241204 #endif
....@@ -1130,7 +1210,9 @@
11301210 	int			(*sendpage)(struct sock *sk, struct page *page,
11311211 					int offset, size_t size, int flags);
11321212 	int			(*bind)(struct sock *sk,
1133
-					struct sockaddr *uaddr, int addr_len);
1213
+					struct sockaddr *addr, int addr_len);
1214
+	int			(*bind_add)(struct sock *sk,
1215
+					struct sockaddr *addr, int addr_len);
11341216 
11351217 	int			(*backlog_rcv) (struct sock *sk,
11361218 						struct sk_buff *skb);
....@@ -1148,7 +1230,7 @@
11481230 	unsigned int		inuse_idx;
11491231 #endif
11501232 
1151
-	bool			(*stream_memory_free)(const struct sock *sk);
1233
+	bool			(*stream_memory_free)(const struct sock *sk, int wake);
11521234 	bool			(*stream_memory_read)(const struct sock *sk);
11531235 	/* Memory pressure */
11541236 	void			(*enter_memory_pressure)(struct sock *sk);
....@@ -1230,19 +1312,29 @@
12301312 #define sk_refcnt_debug_release(sk) do { } while (0)
12311313 #endif /* SOCK_REFCNT_DEBUG */
12321314 
1233
-static inline bool sk_stream_memory_free(const struct sock *sk)
1315
+static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
12341316 {
1235
-	if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1317
+	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
12361318 		return false;
12371319 
12381320 	return sk->sk_prot->stream_memory_free ?
1239
-		sk->sk_prot->stream_memory_free(sk) : true;
1321
+		sk->sk_prot->stream_memory_free(sk, wake) : true;
1322
+}
1323
+
1324
+static inline bool sk_stream_memory_free(const struct sock *sk)
1325
+{
1326
+	return __sk_stream_memory_free(sk, 0);
1327
+}
1328
+
1329
+static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1330
+{
1331
+	return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1332
+	       __sk_stream_memory_free(sk, wake);
12401333 }
12411334 
12421335 static inline bool sk_stream_is_writeable(const struct sock *sk)
12431336 {
1244
-	return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1245
-	       sk_stream_memory_free(sk);
1337
+	return __sk_stream_is_writeable(sk, 0);
12461338 }
12471339 
12481340 static inline int sk_under_cgroup_hierarchy(struct sock *sk,
....@@ -1399,7 +1491,7 @@
13991491 /* sysctl_mem values are in pages, we convert them in SK_MEM_QUANTUM units */
14001492 static inline long sk_prot_mem_limits(const struct sock *sk, int index)
14011493 {
1402
-	long val = sk->sk_prot->sysctl_mem[index];
1494
+	long val = READ_ONCE(sk->sk_prot->sysctl_mem[index]);
14031495 
14041496 #if PAGE_SIZE > SK_MEM_QUANTUM
14051497 	val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
....@@ -1422,19 +1514,23 @@
14221514 
14231515 static inline bool sk_wmem_schedule(struct sock *sk, int size)
14241516 {
1517
+	int delta;
1518
+
14251519 	if (!sk_has_account(sk))
14261520 		return true;
1427
-	return size <= sk->sk_forward_alloc ||
1428
-		__sk_mem_schedule(sk, size, SK_MEM_SEND);
1521
+	delta = size - sk->sk_forward_alloc;
1522
+	return delta <= 0 || __sk_mem_schedule(sk, delta, SK_MEM_SEND);
14291523 }
14301524 
14311525 static inline bool
14321526 sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
14331527 {
1528
+	int delta;
1529
+
14341530 	if (!sk_has_account(sk))
14351531 		return true;
1436
-	return size<= sk->sk_forward_alloc ||
1437
-		__sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1532
+	delta = size - sk->sk_forward_alloc;
1533
+	return delta <= 0 || __sk_mem_schedule(sk, delta, SK_MEM_RECV) ||
14381534 		skb_pfmemalloc(skb);
14391535 }
14401536 
....@@ -1478,11 +1574,18 @@
14781574 		__sk_mem_reclaim(sk, 1 << 20);
14791575 }
14801576 
1577
+DECLARE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
14811578 static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
14821579 {
1483
-	sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1484
-	sk->sk_wmem_queued -= skb->truesize;
1580
+	sk_wmem_queued_add(sk, -skb->truesize);
14851581 	sk_mem_uncharge(sk, skb->truesize);
1582
+	if (static_branch_unlikely(&tcp_tx_skb_cache_key) &&
1583
+	    !sk->sk_tx_skb_cache && !skb_cloned(skb)) {
1584
+		skb_ext_reset(skb);
1585
+		skb_zcopy_clear(skb, true);
1586
+		sk->sk_tx_skb_cache = skb;
1587
+		return;
1588
+	}
14861589 	__kfree_skb(skb);
14871590 }
14881591 
....@@ -1492,7 +1595,7 @@
14921595 		sk->sk_lock.owned = 0;
14931596 
14941597 		/* The sk_lock has mutex_unlock() semantics: */
1495
-		mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
1598
+		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
14961599 	}
14971600 }
14981601 
....@@ -1615,15 +1718,18 @@
16151718 void sock_efree(struct sk_buff *skb);
16161719 #ifdef CONFIG_INET
16171720 void sock_edemux(struct sk_buff *skb);
1721
+void sock_pfree(struct sk_buff *skb);
16181722 #else
16191723 #define sock_edemux sock_efree
16201724 #endif
16211725 
16221726 int sock_setsockopt(struct socket *sock, int level, int op,
1623
-		    char __user *optval, unsigned int optlen);
1727
+		    sockptr_t optval, unsigned int optlen);
16241728 
16251729 int sock_getsockopt(struct socket *sock, int level, int op,
16261730 		    char __user *optval, int __user *optlen);
1731
+int sock_gettstamp(struct socket *sock, void __user *userstamp,
1732
+		   bool timeval, bool time32);
16271733 struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
16281734 				    int noblock, int *errcode);
16291735 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
....@@ -1663,8 +1769,6 @@
16631769 int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
16641770 int sock_no_listen(struct socket *, int);
16651771 int sock_no_shutdown(struct socket *, int);
1666
-int sock_no_getsockopt(struct socket *, int , int, char __user *, int __user *);
1667
-int sock_no_setsockopt(struct socket *, int, int, char __user *, unsigned int);
16681772 int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
16691773 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
16701774 int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
....@@ -1684,11 +1788,7 @@
16841788 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
16851789 			int flags);
16861790 int sock_common_setsockopt(struct socket *sock, int level, int optname,
1687
-				  char __user *optval, unsigned int optlen);
1688
-int compat_sock_common_getsockopt(struct socket *sock, int level,
1689
-		int optname, char __user *optval, int __user *optlen);
1690
-int compat_sock_common_setsockopt(struct socket *sock, int level,
1691
-		int optname, char __user *optval, unsigned int optlen);
1791
+			   sockptr_t optval, unsigned int optlen);
16921792 
16931793 void sk_common_release(struct sock *sk);
16941794 
....@@ -1827,7 +1927,7 @@
18271927 {
18281928 	WARN_ON(parent->sk);
18291929 	write_lock_bh(&sk->sk_callback_lock);
1830
-	rcu_assign_pointer(sk->sk_wq, parent->wq);
1930
+	rcu_assign_pointer(sk->sk_wq, &parent->wq);
18311931 	parent->sk = sk;
18321932 	sk_set_socket(sk, parent);
18331933 	sk->sk_uid = SOCK_INODE(parent)->i_uid;
....@@ -1856,10 +1956,13 @@
18561956 	WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
18571957 }
18581958 
1859
-static inline void sk_rethink_txhash(struct sock *sk)
1959
+static inline bool sk_rethink_txhash(struct sock *sk)
18601960 {
1861
-	if (sk->sk_txhash)
1961
+	if (sk->sk_txhash) {
18621962 		sk_set_txhash(sk);
1963
+		return true;
1964
+	}
1965
+	return false;
18631966 }
18641967 
18651968 static inline struct dst_entry *
....@@ -1882,11 +1985,9 @@
18821985 	return dst;
18831986 }
18841987 
1885
-static inline void dst_negative_advice(struct sock *sk)
1988
+static inline void __dst_negative_advice(struct sock *sk)
18861989 {
18871990 	struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1888
-
1889
-	sk_rethink_txhash(sk);
18901991 
18911992 	if (dst && dst->ops->negative_advice) {
18921993 		ndst = dst->ops->negative_advice(dst);
....@@ -1897,6 +1998,12 @@
18971998 			sk->sk_dst_pending_confirm = 0;
18981999 		}
18992000 	}
2001
+}
2002
+
2003
+static inline void dst_negative_advice(struct sock *sk)
2004
+{
2005
+	sk_rethink_txhash(sk);
2006
+	__dst_negative_advice(sk);
19002007 }
19012008 
19022009 static inline void
....@@ -1941,8 +2048,8 @@
19412048 
19422049 static inline void sk_dst_confirm(struct sock *sk)
19432050 {
1944
-	if (!sk->sk_dst_pending_confirm)
1945
-		sk->sk_dst_pending_confirm = 1;
2051
+	if (!READ_ONCE(sk->sk_dst_pending_confirm))
2052
+		WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
19462053 }
19472054 
19482055 static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
....@@ -1952,10 +2059,10 @@
19522059 		unsigned long now = jiffies;
19532060 
19542061 		/* avoid dirtying neighbour */
1955
-		if (n->confirmed != now)
1956
-			n->confirmed = now;
1957
-		if (sk && sk->sk_dst_pending_confirm)
1958
-			sk->sk_dst_pending_confirm = 0;
2062
+		if (READ_ONCE(n->confirmed) != now)
2063
+			WRITE_ONCE(n->confirmed, now);
2064
+		if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2065
+			WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
19592066 	}
19602067 }
19612068 
....@@ -2020,7 +2127,7 @@
20202127 	skb->len	     += copy;
20212128 	skb->data_len	     += copy;
20222129 	skb->truesize	     += copy;
2023
-	sk->sk_wmem_queued   += copy;
2130
+	sk_wmem_queued_add(sk, copy);
20242131 	sk_mem_charge(sk, copy);
20252132 	return 0;
20262133 }
....@@ -2029,7 +2136,7 @@
20292136  * sk_wmem_alloc_get - returns write allocations
20302137  * @sk: socket
20312138  *
2032
- * Returns sk_wmem_alloc minus initial offset of one
2139
+ * Return: sk_wmem_alloc minus initial offset of one
20332140  */
20342141 static inline int sk_wmem_alloc_get(const struct sock *sk)
20352142 {
....@@ -2040,7 +2147,7 @@
20402147  * sk_rmem_alloc_get - returns read allocations
20412148  * @sk: socket
20422149  *
2043
- * Returns sk_rmem_alloc
2150
+ * Return: sk_rmem_alloc
20442151  */
20452152 static inline int sk_rmem_alloc_get(const struct sock *sk)
20462153 {
....@@ -2051,7 +2158,7 @@
20512158  * sk_has_allocations - check if allocations are outstanding
20522159  * @sk: socket
20532160  *
2054
- * Returns true if socket has write or read allocations
2161
+ * Return: true if socket has write or read allocations
20552162  */
20562163 static inline bool sk_has_allocations(const struct sock *sk)
20572164 {
....@@ -2062,7 +2169,7 @@
20622169  * skwq_has_sleeper - check if there are any waiting processes
20632170  * @wq: struct socket_wq
20642171  *
2065
- * Returns true if socket_wq has waiting processes
2172
+ * Return: true if socket_wq has waiting processes
20662173  *
20672174  * The purpose of the skwq_has_sleeper and sock_poll_wait is to wrap the memory
20682175  * barrier call. They were added due to the race found within the tcp code.
....@@ -2101,18 +2208,12 @@
21012208  * @p:              poll_table
21022209  *
21032210  * See the comments in the wq_has_sleeper function.
2104
- *
2105
- * Do not derive sock from filp->private_data here. An SMC socket establishes
2106
- * an internal TCP socket that is used in the fallback case. All socket
2107
- * operations on the SMC socket are then forwarded to the TCP socket. In case of
2108
- * poll, the filp->private_data pointer references the SMC socket because the
2109
- * TCP socket has no file assigned.
21102211  */
21112212 static inline void sock_poll_wait(struct file *filp, struct socket *sock,
21122213 				  poll_table *p)
21132214 {
21142215 	if (!poll_does_not_wait(p)) {
2115
-		poll_wait(filp, &sock->wq->wait, p);
2216
+		poll_wait(filp, &sock->wq.wait, p);
21162217 		/* We need to be sure we are in sync with the
21172218 		 * socket flags modification.
21182219 		 *
....@@ -2152,10 +2253,23 @@
21522253 	sk_mem_charge(sk, skb->truesize);
21532254 }
21542255 
2256
+static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2257
+{
2258
+	if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
2259
+		skb_orphan(skb);
2260
+		skb->destructor = sock_efree;
2261
+		skb->sk = sk;
2262
+		return true;
2263
+	}
2264
+	return false;
2265
+}
2266
+
21552267 void sk_reset_timer(struct sock *sk, struct timer_list *timer,
21562268 		    unsigned long expires);
21572269 
21582270 void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2271
+
2272
+void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
21592273 
21602274 int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
21612275 			struct sk_buff *skb, unsigned int flags,
....@@ -2174,8 +2288,13 @@
21742288 static inline int sock_error(struct sock *sk)
21752289 {
21762290 	int err;
2177
-	if (likely(!sk->sk_err))
2291
+
2292
+	/* Avoid an atomic operation for the common case.
2293
+	 * This is racy since another cpu/thread can change sk_err under us.
2294
+	 */
2295
+	if (likely(data_race(!sk->sk_err)))
21782296 		return 0;
2297
+
21792298 	err = xchg(&sk->sk_err, 0);
21802299 	return -err;
21812300 }
....@@ -2235,10 +2354,14 @@
22352354 
22362355 static inline void sk_stream_moderate_sndbuf(struct sock *sk)
22372356 {
2238
-	if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2239
-		sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2240
-		sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2241
-	}
2357
+	u32 val;
2358
+
2359
+	if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2360
+		return;
2361
+
2362
+	val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2363
+
2364
+	WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
22422365 }
22432366 
22442367 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
....@@ -2249,16 +2372,22 @@
22492372  * @sk: socket
22502373  *
22512374  * Use the per task page_frag instead of the per socket one for
2252
- * optimization when we know that we're in the normal context and owns
2375
+ * optimization when we know that we're in process context and own
22532376  * everything that's associated with %current.
22542377  *
2255
- * gfpflags_allow_blocking() isn't enough here as direct reclaim may nest
2256
- * inside other socket operations and end up recursing into sk_page_frag()
2257
- * while it's already in use.
2378
+ * Both direct reclaim and page faults can nest inside other
2379
+ * socket operations and end up recursing into sk_page_frag()
2380
+ * while it's already in use: explicitly avoid task page_frag
2381
+ * usage if the caller is potentially doing any of them.
2382
+ * This assumes that page fault handlers use the GFP_NOFS flags.
2383
+ *
2384
+ * Return: a per task page_frag if context allows that,
2385
+ * otherwise a per socket one.
22582386  */
22592387 static inline struct page_frag *sk_page_frag(struct sock *sk)
22602388 {
2261
-	if (gfpflags_normal_context(sk->sk_allocation))
2389
+	if ((sk->sk_allocation & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC | __GFP_FS)) ==
2390
+	    (__GFP_DIRECT_RECLAIM | __GFP_FS))
22622391 		return &current->task_frag;
22632392 
22642393 	return &sk->sk_frag;
....@@ -2266,16 +2395,12 @@
22662395 
22672396 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
22682397 
2269
-int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
2270
-		int sg_start, int *sg_curr, unsigned int *sg_size,
2271
-		int first_coalesce);
2272
-
22732398 /*
22742399  *	Default write policy as shown to user space via poll/select/SIGIO
22752400  */
22762401 static inline bool sock_writeable(const struct sock *sk)
22772402 {
2278
-	return refcount_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2403
+	return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
22792404 }
22802405 
22812406 static inline gfp_t gfp_any(void)
....@@ -2295,7 +2420,9 @@
22952420 
22962421 static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
22972422 {
2298
-	return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2423
+	int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2424
+
2425
+	return v ?: 1;
22992426 }
23002427 
23012428 /* Alas, with timeout socket operations are not restartable.
....@@ -2314,7 +2441,7 @@
23142441  * using skb->cb[] would keep using it directly and utilize its
23152442  * alignement guarantee.
23162443  */
2317
-#define SOCK_SKB_CB_OFFSET ((FIELD_SIZEOF(struct sk_buff, cb) - \
2444
+#define SOCK_SKB_CB_OFFSET ((sizeof_field(struct sk_buff, cb) - \
23182445 			    sizeof(struct sock_skb_cb)))
23192446 
23202447 #define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
....@@ -2418,22 +2545,40 @@
24182545 void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
24192546 
24202547 /**
2421
- * sock_tx_timestamp - checks whether the outgoing packet is to be time stamped
2548
+ * _sock_tx_timestamp - checks whether the outgoing packet is to be time stamped
24222549  * @sk:		socket sending this packet
24232550  * @tsflags:	timestamping flags to use
24242551  * @tx_flags:	completed with instructions for time stamping
2552
+ * @tskey:      filled in with next sk_tskey (not for TCP, which uses seqno)
24252553  *
24262554  * Note: callers should take care of initial ``*tx_flags`` value (usually 0)
24272555  */
2428
-static inline void sock_tx_timestamp(const struct sock *sk, __u16 tsflags,
2429
-				     __u8 *tx_flags)
2556
+static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2557
+				      __u8 *tx_flags, __u32 *tskey)
24302558 {
2431
-	if (unlikely(tsflags))
2559
+	if (unlikely(tsflags)) {
24322560 		__sock_tx_timestamp(tsflags, tx_flags);
2561
+		if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2562
+		    tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
2563
+			*tskey = sk->sk_tskey++;
2564
+	}
24332565 	if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
24342566 		*tx_flags |= SKBTX_WIFI_STATUS;
24352567 }
24362568 
2569
+static inline void sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2570
+				     __u8 *tx_flags)
2571
+{
2572
+	_sock_tx_timestamp(sk, tsflags, tx_flags, NULL);
2573
+}
2574
+
2575
+static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
2576
+{
2577
+	_sock_tx_timestamp(skb->sk, tsflags, &skb_shinfo(skb)->tx_flags,
2578
+			   &skb_shinfo(skb)->tskey);
2579
+}
2580
+
2581
+DECLARE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
24372582 /**
24382583  * sk_eat_skb - Release a skb if it is no longer needed
24392584  * @sk: socket to eat this skb from
....@@ -2445,6 +2590,12 @@
24452590 static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
24462591 {
24472592 	__skb_unlink(skb, &sk->sk_receive_queue);
2593
+	if (static_branch_unlikely(&tcp_rx_skb_cache_key) &&
2594
+	    !sk->sk_rx_skb_cache) {
2595
+		sk->sk_rx_skb_cache = skb;
2596
+		skb_orphan(skb);
2597
+		return;
2598
+	}
24482599 	__kfree_skb(skb);
24492600 }
24502601 
....@@ -2460,16 +2611,14 @@
24602611 	write_pnet(&sk->sk_net, net);
24612612 }
24622613 
2463
-static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2614
+static inline bool
2615
+skb_sk_is_prefetched(struct sk_buff *skb)
24642616 {
2465
-	if (skb->sk) {
2466
-		struct sock *sk = skb->sk;
2467
-
2468
-		skb->destructor = NULL;
2469
-		skb->sk = NULL;
2470
-		return sk;
2471
-	}
2472
-	return NULL;
2617
+#ifdef CONFIG_INET
2618
+	return skb->destructor == sock_pfree;
2619
+#else
2620
+	return false;
2621
+#endif /* CONFIG_INET */
24732622 }
24742623 
24752624 /* This helper checks if a socket is a full socket,
....@@ -2480,8 +2629,38 @@
24802629 	return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
24812630 }
24822631 
2632
+static inline bool
2633
+sk_is_refcounted(struct sock *sk)
2634
+{
2635
+	/* Only full sockets have sk->sk_flags. */
2636
+	return !sk_fullsock(sk) || !sock_flag(sk, SOCK_RCU_FREE);
2637
+}
2638
+
2639
+/**
2640
+ * skb_steal_sock - steal a socket from an sk_buff
2641
+ * @skb: sk_buff to steal the socket from
2642
+ * @refcounted: is set to true if the socket is reference-counted
2643
+ */
2644
+static inline struct sock *
2645
+skb_steal_sock(struct sk_buff *skb, bool *refcounted)
2646
+{
2647
+	if (skb->sk) {
2648
+		struct sock *sk = skb->sk;
2649
+
2650
+		*refcounted = true;
2651
+		if (skb_sk_is_prefetched(skb))
2652
+			*refcounted = sk_is_refcounted(sk);
2653
+		skb->destructor = NULL;
2654
+		skb->sk = NULL;
2655
+		return sk;
2656
+	}
2657
+	*refcounted = false;
2658
+	return NULL;
2659
+}
2660
+
24832661 /* Checks if this SKB belongs to an HW offloaded socket
24842662  * and whether any SW fallbacks are required based on dev.
2663
+ * Check decrypted mark in case skb_orphan() cleared socket.
24852664  */
24862665 static inline struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
24872666 						   struct net_device *dev)
....@@ -2489,8 +2668,15 @@
24892668 #ifdef CONFIG_SOCK_VALIDATE_XMIT
24902669 	struct sock *sk = skb->sk;
24912670 
2492
-	if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb)
2671
+	if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb) {
24932672 		skb = sk->sk_validate_xmit_skb(sk, dev, skb);
2673
+#ifdef CONFIG_TLS_DEVICE
2674
+	} else if (unlikely(skb->decrypted)) {
2675
+		pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
2676
+		kfree_skb(skb);
2677
+		skb = NULL;
2678
+#endif
2679
+	}
24942680 #endif
24952681 
24962682 	return skb;
....@@ -2504,9 +2690,7 @@
25042690 	return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
25052691 }
25062692 
2507
-void sock_enable_timestamp(struct sock *sk, int flag);
2508
-int sock_get_timestamp(struct sock *, struct timeval __user *);
2509
-int sock_get_timestampns(struct sock *, struct timespec __user *);
2693
+void sock_enable_timestamp(struct sock *sk, enum sock_flags flag);
25102694 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
25112695 		       int type);
25122696 
....@@ -2536,22 +2720,25 @@
25362720 extern __u32 sysctl_wmem_default;
25372721 extern __u32 sysctl_rmem_default;
25382722 
2723
+#define SKB_FRAG_PAGE_ORDER	get_order(32768)
2724
+DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2725
+
25392726 static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
25402727 {
25412728 	/* Does this proto have per netns sysctl_wmem ? */
25422729 	if (proto->sysctl_wmem_offset)
2543
-		return *(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset);
2730
+		return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset));
25442731 
2545
-	return *proto->sysctl_wmem;
2732
+	return READ_ONCE(*proto->sysctl_wmem);
25462733 }
25472734 
25482735 static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
25492736 {
25502737 	/* Does this proto have per netns sysctl_rmem ? */
25512738 	if (proto->sysctl_rmem_offset)
2552
-		return *(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset);
2739
+		return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset));
25532740 
2554
-	return *proto->sysctl_rmem;
2741
+	return READ_ONCE(*proto->sysctl_rmem);
25552742 }
25562743 
25572744 /* Default TCP Small queue budget is ~1 ms of data (1sec >> 10)
....@@ -2560,9 +2747,9 @@
25602747  */
25612748 static inline void sk_pacing_shift_update(struct sock *sk, int val)
25622749 {
2563
-	if (!sk || !sk_fullsock(sk) || sk->sk_pacing_shift == val)
2750
+	if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
25642751 		return;
2565
-	sk->sk_pacing_shift = val;
2752
+	WRITE_ONCE(sk->sk_pacing_shift, val);
25662753 }
25672754 
25682755 /* if a socket is bound to a device, check that the given device
....@@ -2584,4 +2771,19 @@
25842771 	return false;
25852772 }
25862773 
2774
+void sock_def_readable(struct sock *sk);
2775
+
2776
+int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk);
2777
+void sock_enable_timestamps(struct sock *sk);
2778
+void sock_no_linger(struct sock *sk);
2779
+void sock_set_keepalive(struct sock *sk);
2780
+void sock_set_priority(struct sock *sk, u32 priority);
2781
+void sock_set_rcvbuf(struct sock *sk, int val);
2782
+void sock_set_mark(struct sock *sk, u32 val);
2783
+void sock_set_reuseaddr(struct sock *sk);
2784
+void sock_set_reuseport(struct sock *sk);
2785
+void sock_set_sndtimeo(struct sock *sk, s64 secs);
2786
+
2787
+int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len);
2788
+
25872789 #endif	/* _SOCK_H */