~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Routines having to do with the 'struct sk_buff' memory handlers.
3	4	*
..	..	@@ -25,11 +26,6 @@
25	26	* disabled, or you better be real sure that the operation is atomic
26	27	* with respect to whatever list is being frobbed (e.g. via lock_sock()
27	28	* or via disabling bottom half handlers, etc).
28		- *
29		- * This program is free software; you can redistribute it and/or
30		- * modify it under the terms of the GNU General Public License
31		- * as published by the Free Software Foundation; either version
32		- * 2 of the License, or (at your option) any later version.
33	29	*/
34	30
35	31	/*
..	..	@@ -63,6 +59,7 @@
63	59	#include <linux/errqueue.h>
64	60	#include <linux/prefetch.h>
65	61	#include <linux/if_vlan.h>
	62	+#include <linux/mpls.h>
66	63
67	64	#include <net/protocol.h>
68	65	#include <net/dst.h>
..	..	@@ -70,15 +67,24 @@
70	67	#include <net/checksum.h>
71	68	#include <net/ip6_checksum.h>
72	69	#include <net/xfrm.h>
	70	+#include <net/mpls.h>
	71	+#include <net/mptcp.h>
73	72
74	73	#include <linux/uaccess.h>
75	74	#include <trace/events/skb.h>
76	75	#include <linux/highmem.h>
77	76	#include <linux/capability.h>
78	77	#include <linux/user_namespace.h>
	78	+#include <linux/indirect_call_wrapper.h>
	79	+#include <trace/hooks/net.h>
	80	+
	81	+#include "datagram.h"
79	82
80	83	struct kmem_cache *skbuff_head_cache __ro_after_init;
81	84	static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
	85	+#ifdef CONFIG_SKB_EXTENSIONS
	86	+static struct kmem_cache *skbuff_ext_cache __ro_after_init;
	87	+#endif
82	88	int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
83	89	EXPORT_SYMBOL(sysctl_max_skb_frags);
84	90
..	..	@@ -97,7 +103,7 @@
97	103	static void skb_panic(struct sk_buff skb, unsigned int sz, void addr,
98	104	const char msg[])
99	105	{
100		- pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
	106	+ pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
101	107	msg, addr, skb->len, sz, skb->head, skb->data,
102	108	(unsigned long)skb->tail, (unsigned long)skb->end,
103	109	skb->dev ? skb->dev->name : "<NULL>");
..	..	@@ -244,6 +250,9 @@
244	250
245	251	fclones->skb2.fclone = SKB_FCLONE_CLONE;
246	252	}
	253	+
	254	+ skb_set_kcov_handle(skb, kcov_common_handle());
	255	+
247	256	out:
248	257	return skb;
249	258	nodata:
..	..	@@ -252,6 +261,35 @@
252	261	goto out;
253	262	}
254	263	EXPORT_SYMBOL(__alloc_skb);
	264	+
	265	+/* Caller must provide SKB that is memset cleared */
	266	+static struct sk_buff __build_skb_around(struct sk_buff skb,
	267	+ void *data, unsigned int frag_size)
	268	+{
	269	+ struct skb_shared_info *shinfo;
	270	+ unsigned int size = frag_size ? : ksize(data);
	271	+
	272	+ size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	273	+
	274	+ /* Assumes caller memset cleared SKB */
	275	+ skb->truesize = SKB_TRUESIZE(size);
	276	+ refcount_set(&skb->users, 1);
	277	+ skb->head = data;
	278	+ skb->data = data;
	279	+ skb_reset_tail_pointer(skb);
	280	+ skb->end = skb->tail + size;
	281	+ skb->mac_header = (typeof(skb->mac_header))~0U;
	282	+ skb->transport_header = (typeof(skb->transport_header))~0U;
	283	+
	284	+ /* make sure we initialize shinfo sequentially */
	285	+ shinfo = skb_shinfo(skb);
	286	+ memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
	287	+ atomic_set(&shinfo->dataref, 1);
	288	+
	289	+ skb_set_kcov_handle(skb, kcov_common_handle());
	290	+
	291	+ return skb;
	292	+}
255	293
256	294	/**
257	295	* __build_skb - build a network buffer
..	..	@@ -274,32 +312,15 @@
274	312	*/
275	313	struct sk_buff __build_skb(void data, unsigned int frag_size)
276	314	{
277		- struct skb_shared_info *shinfo;
278	315	struct sk_buff *skb;
279		- unsigned int size = frag_size ? : ksize(data);
280	316
281	317	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
282		- if (!skb)
	318	+ if (unlikely(!skb))
283	319	return NULL;
284	320
285		- size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
286		-
287	321	memset(skb, 0, offsetof(struct sk_buff, tail));
288		- skb->truesize = SKB_TRUESIZE(size);
289		- refcount_set(&skb->users, 1);
290		- skb->head = data;
291		- skb->data = data;
292		- skb_reset_tail_pointer(skb);
293		- skb->end = skb->tail + size;
294		- skb->mac_header = (typeof(skb->mac_header))~0U;
295		- skb->transport_header = (typeof(skb->transport_header))~0U;
296	322
297		- /* make sure we initialize shinfo sequentially */
298		- shinfo = skb_shinfo(skb);
299		- memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
300		- atomic_set(&shinfo->dataref, 1);
301		-
302		- return skb;
	323	+ return __build_skb_around(skb, data, frag_size);
303	324	}
304	325
305	326	/* build_skb() is wrapper over __build_skb(), that specifically
..	..	@@ -320,6 +341,29 @@
320	341	}
321	342	EXPORT_SYMBOL(build_skb);
322	343
	344	+/**
	345	+ * build_skb_around - build a network buffer around provided skb
	346	+ * @skb: sk_buff provide by caller, must be memset cleared
	347	+ * @data: data buffer provided by caller
	348	+ * @frag_size: size of data, or 0 if head was kmalloced
	349	+ */
	350	+struct sk_buff build_skb_around(struct sk_buff skb,
	351	+ void *data, unsigned int frag_size)
	352	+{
	353	+ if (unlikely(!skb))
	354	+ return NULL;
	355	+
	356	+ skb = __build_skb_around(skb, data, frag_size);
	357	+
	358	+ if (skb && frag_size) {
	359	+ skb->head_frag = 1;
	360	+ if (page_is_pfmemalloc(virt_to_head_page(data)))
	361	+ skb->pfmemalloc = 1;
	362	+ }
	363	+ return skb;
	364	+}
	365	+EXPORT_SYMBOL(build_skb_around);
	366	+
323	367	#define NAPI_SKB_CACHE_SIZE 64
324	368
325	369	struct napi_alloc_cache {
..	..	@@ -330,34 +374,6 @@
330	374
331	375	static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
332	376	static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
333		-
334		-static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
335		-{
336		- struct page_frag_cache *nc;
337		- unsigned long flags;
338		- void *data;
339		-
340		- local_irq_save(flags);
341		- nc = this_cpu_ptr(&netdev_alloc_cache);
342		- data = page_frag_alloc(nc, fragsz, gfp_mask);
343		- local_irq_restore(flags);
344		- return data;
345		-}
346		-
347		-/**
348		- * netdev_alloc_frag - allocate a page fragment
349		- * @fragsz: fragment size
350		- *
351		- * Allocates a frag from a page for receive buffer.
352		- * Uses GFP_ATOMIC allocations.
353		- */
354		-void *netdev_alloc_frag(unsigned int fragsz)
355		-{
356		- fragsz = SKB_DATA_ALIGN(fragsz);
357		-
358		- return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
359		-}
360		-EXPORT_SYMBOL(netdev_alloc_frag);
361	377
362	378	static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
363	379	{
..	..	@@ -373,6 +389,31 @@
373	389	return __napi_alloc_frag(fragsz, GFP_ATOMIC);
374	390	}
375	391	EXPORT_SYMBOL(napi_alloc_frag);
	392	+
	393	+/**
	394	+ * netdev_alloc_frag - allocate a page fragment
	395	+ * @fragsz: fragment size
	396	+ *
	397	+ * Allocates a frag from a page for receive buffer.
	398	+ * Uses GFP_ATOMIC allocations.
	399	+ */
	400	+void *netdev_alloc_frag(unsigned int fragsz)
	401	+{
	402	+ struct page_frag_cache *nc;
	403	+ void *data;
	404	+
	405	+ fragsz = SKB_DATA_ALIGN(fragsz);
	406	+ if (in_irq() \|\| irqs_disabled()) {
	407	+ nc = this_cpu_ptr(&netdev_alloc_cache);
	408	+ data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
	409	+ } else {
	410	+ local_bh_disable();
	411	+ data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
	412	+ local_bh_enable();
	413	+ }
	414	+ return data;
	415	+}
	416	+EXPORT_SYMBOL(netdev_alloc_frag);
376	417
377	418	/**
378	419	* __netdev_alloc_skb - allocate an skbuff for rx on a specific device
..	..	@@ -391,7 +432,6 @@
391	432	gfp_t gfp_mask)
392	433	{
393	434	struct page_frag_cache *nc;
394		- unsigned long flags;
395	435	struct sk_buff *skb;
396	436	bool pfmemalloc;
397	437	void *data;
..	..	@@ -416,13 +456,17 @@
416	456	if (sk_memalloc_socks())
417	457	gfp_mask \|= __GFP_MEMALLOC;
418	458
419		- local_irq_save(flags);
420		-
421		- nc = this_cpu_ptr(&netdev_alloc_cache);
422		- data = page_frag_alloc(nc, len, gfp_mask);
423		- pfmemalloc = nc->pfmemalloc;
424		-
425		- local_irq_restore(flags);
	459	+ if (in_irq() \|\| irqs_disabled()) {
	460	+ nc = this_cpu_ptr(&netdev_alloc_cache);
	461	+ data = page_frag_alloc(nc, len, gfp_mask);
	462	+ pfmemalloc = nc->pfmemalloc;
	463	+ } else {
	464	+ local_bh_disable();
	465	+ nc = this_cpu_ptr(&napi_alloc_cache.page);
	466	+ data = page_frag_alloc(nc, len, gfp_mask);
	467	+ pfmemalloc = nc->pfmemalloc;
	468	+ local_bh_enable();
	469	+ }
426	470
427	471	if (unlikely(!data))
428	472	return NULL;
..	..	@@ -433,7 +477,6 @@
433	477	return NULL;
434	478	}
435	479
436		- /* use OR instead of assignment to avoid clearing of bits in mask */
437	480	if (pfmemalloc)
438	481	skb->pfmemalloc = 1;
439	482	skb->head_frag = 1;
..	..	@@ -498,7 +541,6 @@
498	541	return NULL;
499	542	}
500	543
501		- /* use OR instead of assignment to avoid clearing of bits in mask */
502	544	if (nc->page.pfmemalloc)
503	545	skb->pfmemalloc = 1;
504	546	skb->head_frag = 1;
..	..	@@ -619,7 +661,6 @@
619	661	void skb_release_head_state(struct sk_buff *skb)
620	662	{
621	663	skb_dst_drop(skb);
622		- secpath_reset(skb);
623	664	if (skb->destructor) {
624	665	WARN_ON(in_irq());
625	666	skb->destructor(skb);
..	..	@@ -627,9 +668,7 @@
627	668	#if IS_ENABLED(CONFIG_NF_CONNTRACK)
628	669	nf_conntrack_put(skb_nfct(skb));
629	670	#endif
630		-#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
631		- nf_bridge_put(skb->nf_bridge);
632		-#endif
	671	+ skb_ext_put(skb);
633	672	}
634	673
635	674	/* Free everything but the sk_buff shell. */
..	..	@@ -668,6 +707,7 @@
668	707	if (!skb_unref(skb))
669	708	return;
670	709
	710	+ trace_android_vh_kfree_skb(skb);
671	711	trace_kfree_skb(skb, __builtin_return_address(0));
672	712	__kfree_skb(skb);
673	713	}
..	..	@@ -684,6 +724,101 @@
684	724	}
685	725	EXPORT_SYMBOL(kfree_skb_list);
686	726
	727	+/* Dump skb information and contents.
	728	+ *
	729	+ * Must only be called from net_ratelimit()-ed paths.
	730	+ *
	731	+ * Dumps whole packets if full_pkt, only headers otherwise.
	732	+ */
	733	+void skb_dump(const char level, const struct sk_buff skb, bool full_pkt)
	734	+{
	735	+ struct skb_shared_info *sh = skb_shinfo(skb);
	736	+ struct net_device *dev = skb->dev;
	737	+ struct sock *sk = skb->sk;
	738	+ struct sk_buff *list_skb;
	739	+ bool has_mac, has_trans;
	740	+ int headroom, tailroom;
	741	+ int i, len, seg_len;
	742	+
	743	+ if (full_pkt)
	744	+ len = skb->len;
	745	+ else
	746	+ len = min_t(int, skb->len, MAX_HEADER + 128);
	747	+
	748	+ headroom = skb_headroom(skb);
	749	+ tailroom = skb_tailroom(skb);
	750	+
	751	+ has_mac = skb_mac_header_was_set(skb);
	752	+ has_trans = skb_transport_header_was_set(skb);
	753	+
	754	+ printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
	755	+ "mac=(%d,%d) net=(%d,%d) trans=%d\n"
	756	+ "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
	757	+ "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
	758	+ "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
	759	+ level, skb->len, headroom, skb_headlen(skb), tailroom,
	760	+ has_mac ? skb->mac_header : -1,
	761	+ has_mac ? skb_mac_header_len(skb) : -1,
	762	+ skb->network_header,
	763	+ has_trans ? skb_network_header_len(skb) : -1,
	764	+ has_trans ? skb->transport_header : -1,
	765	+ sh->tx_flags, sh->nr_frags,
	766	+ sh->gso_size, sh->gso_type, sh->gso_segs,
	767	+ skb->csum, skb->ip_summed, skb->csum_complete_sw,
	768	+ skb->csum_valid, skb->csum_level,
	769	+ skb->hash, skb->sw_hash, skb->l4_hash,
	770	+ ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
	771	+
	772	+ if (dev)
	773	+ printk("%sdev name=%s feat=%pNF\n",
	774	+ level, dev->name, &dev->features);
	775	+ if (sk)
	776	+ printk("%ssk family=%hu type=%u proto=%u\n",
	777	+ level, sk->sk_family, sk->sk_type, sk->sk_protocol);
	778	+
	779	+ if (full_pkt && headroom)
	780	+ print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
	781	+ 16, 1, skb->head, headroom, false);
	782	+
	783	+ seg_len = min_t(int, skb_headlen(skb), len);
	784	+ if (seg_len)
	785	+ print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
	786	+ 16, 1, skb->data, seg_len, false);
	787	+ len -= seg_len;
	788	+
	789	+ if (full_pkt && tailroom)
	790	+ print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
	791	+ 16, 1, skb_tail_pointer(skb), tailroom, false);
	792	+
	793	+ for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
	794	+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
	795	+ u32 p_off, p_len, copied;
	796	+ struct page *p;
	797	+ u8 *vaddr;
	798	+
	799	+ skb_frag_foreach_page(frag, skb_frag_off(frag),
	800	+ skb_frag_size(frag), p, p_off, p_len,
	801	+ copied) {
	802	+ seg_len = min_t(int, p_len, len);
	803	+ vaddr = kmap_atomic(p);
	804	+ print_hex_dump(level, "skb frag: ",
	805	+ DUMP_PREFIX_OFFSET,
	806	+ 16, 1, vaddr + p_off, seg_len, false);
	807	+ kunmap_atomic(vaddr);
	808	+ len -= seg_len;
	809	+ if (!len)
	810	+ break;
	811	+ }
	812	+ }
	813	+
	814	+ if (full_pkt && skb_has_frag_list(skb)) {
	815	+ printk("skb fraglist:\n");
	816	+ skb_walk_frags(skb, list_skb)
	817	+ skb_dump(level, list_skb, true);
	818	+ }
	819	+}
	820	+EXPORT_SYMBOL(skb_dump);
	821	+
687	822	/**
688	823	* skb_tx_error - report an sk_buff xmit error
689	824	* @skb: buffer that triggered an error
..	..	@@ -697,6 +832,7 @@
697	832	}
698	833	EXPORT_SYMBOL(skb_tx_error);
699	834
	835	+#ifdef CONFIG_TRACEPOINTS
700	836	/**
701	837	* consume_skb - free an skbuff
702	838	* @skb: buffer to free
..	..	@@ -714,6 +850,7 @@
714	850	__kfree_skb(skb);
715	851	}
716	852	EXPORT_SYMBOL(consume_skb);
	853	+#endif
717	854
718	855	/**
719	856	* consume_stateless_skb - free an skbuff, assuming it is stateless
..	..	@@ -770,9 +907,6 @@
770	907
771	908	void napi_consume_skb(struct sk_buff *skb, int budget)
772	909	{
773		- if (unlikely(!skb))
774		- return;
775		-
776	910	/* Zero budget indicate non-NAPI context called us, like netpoll */
777	911	if (unlikely(!budget)) {
778	912	dev_consume_skb_any(skb);
..	..	@@ -809,9 +943,7 @@
809	943	new->dev = old->dev;
810	944	memcpy(new->cb, old->cb, sizeof(old->cb));
811	945	skb_dst_copy(new, old);
812		-#ifdef CONFIG_XFRM
813		- new->sp = secpath_get(old->sp);
814		-#endif
	946	+ __skb_ext_copy(new, old);
815	947	__nf_copy(new, old, false);
816	948
817	949	/* Note : this field could be in headers_start/headers_end section
..	..	@@ -887,6 +1019,31 @@
887	1019	return n;
888	1020	#undef C
889	1021	}
	1022	+
	1023	+/**
	1024	+ * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
	1025	+ * @first: first sk_buff of the msg
	1026	+ */
	1027	+struct sk_buff alloc_skb_for_msg(struct sk_buff first)
	1028	+{
	1029	+ struct sk_buff *n;
	1030	+
	1031	+ n = alloc_skb(0, GFP_ATOMIC);
	1032	+ if (!n)
	1033	+ return NULL;
	1034	+
	1035	+ n->len = first->len;
	1036	+ n->data_len = first->len;
	1037	+ n->truesize = first->truesize;
	1038	+
	1039	+ skb_shinfo(n)->frag_list = first;
	1040	+
	1041	+ __copy_skb_header(n, first);
	1042	+ n->destructor = NULL;
	1043	+
	1044	+ return n;
	1045	+}
	1046	+EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
890	1047
891	1048	/**
892	1049	* skb_morph - morph one skb into another
..	..	@@ -1012,7 +1169,11 @@
1012	1169	uarg->len++;
1013	1170	uarg->bytelen = bytelen;
1014	1171	atomic_set(&sk->sk_zckey, ++next);
1015		- sock_zerocopy_get(uarg);
	1172	+
	1173	+ /* no extra ref when appending to datagram (MSG_MORE) */
	1174	+ if (sk->sk_type == SOCK_STREAM)
	1175	+ sock_zerocopy_get(uarg);
	1176	+
1016	1177	return uarg;
1017	1178	}
1018	1179	}
..	..	@@ -1102,7 +1263,7 @@
1102	1263	}
1103	1264	EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1104	1265
1105		-void sock_zerocopy_put_abort(struct ubuf_info *uarg)
	1266	+void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1106	1267	{
1107	1268	if (uarg) {
1108	1269	struct sock *sk = skb_from_uarg(uarg)->sk;
..	..	@@ -1110,13 +1271,17 @@
1110	1271	atomic_dec(&sk->sk_zckey);
1111	1272	uarg->len--;
1112	1273
1113		- sock_zerocopy_put(uarg);
	1274	+ if (have_uref)
	1275	+ sock_zerocopy_put(uarg);
1114	1276	}
1115	1277	}
1116	1278	EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1117	1279
1118		-extern int __zerocopy_sg_from_iter(struct sock sk, struct sk_buff skb,
1119		- struct iov_iter *from, size_t length);
	1280	+int skb_zerocopy_iter_dgram(struct sk_buff skb, struct msghdr msg, int len)
	1281	+{
	1282	+ return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
	1283	+}
	1284	+EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1120	1285
1121	1286	int skb_zerocopy_iter_stream(struct sock sk, struct sk_buff skb,
1122	1287	struct msghdr *msg, int len,
..	..	@@ -1144,7 +1309,7 @@
1144	1309	return err;
1145	1310	}
1146	1311
1147		- skb_zcopy_set(skb, uarg);
	1312	+ skb_zcopy_set(skb, uarg, NULL);
1148	1313	return skb->len - orig_len;
1149	1314	}
1150	1315	EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
..	..	@@ -1164,7 +1329,7 @@
1164	1329	if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1165	1330	return -EIO;
1166	1331	}
1167		- skb_zcopy_set(nskb, skb_uarg(orig));
	1332	+ skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1168	1333	}
1169	1334	return 0;
1170	1335	}
..	..	@@ -1220,7 +1385,7 @@
1220	1385	struct page *p;
1221	1386	u8 *vaddr;
1222	1387
1223		- skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
	1388	+ skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1224	1389	p, p_off, p_len, copied) {
1225	1390	u32 copy, done = 0;
1226	1391	vaddr = kmap_atomic(p);
..	..	@@ -1510,11 +1675,10 @@
1510	1675	skb->head = data;
1511	1676	skb->head_frag = 0;
1512	1677	skb->data += off;
	1678	+
	1679	+ skb_set_end_offset(skb, size);
1513	1680	#ifdef NET_SKBUFF_DATA_USES_OFFSET
1514		- skb->end = size;
1515	1681	off = nhead;
1516		-#else
1517		- skb->end = skb->head + size;
1518	1682	#endif
1519	1683	skb->tail += off;
1520	1684	skb_headers_offset_update(skb, nhead);
..	..	@@ -1561,6 +1725,38 @@
1561	1725	return skb2;
1562	1726	}
1563	1727	EXPORT_SYMBOL(skb_realloc_headroom);
	1728	+
	1729	+int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
	1730	+{
	1731	+ unsigned int saved_end_offset, saved_truesize;
	1732	+ struct skb_shared_info *shinfo;
	1733	+ int res;
	1734	+
	1735	+ saved_end_offset = skb_end_offset(skb);
	1736	+ saved_truesize = skb->truesize;
	1737	+
	1738	+ res = pskb_expand_head(skb, 0, 0, pri);
	1739	+ if (res)
	1740	+ return res;
	1741	+
	1742	+ skb->truesize = saved_truesize;
	1743	+
	1744	+ if (likely(skb_end_offset(skb) == saved_end_offset))
	1745	+ return 0;
	1746	+
	1747	+ shinfo = skb_shinfo(skb);
	1748	+
	1749	+ /* We are about to change back skb->end,
	1750	+ * we need to move skb_shinfo() to its new location.
	1751	+ */
	1752	+ memmove(skb->head + saved_end_offset,
	1753	+ shinfo,
	1754	+ offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
	1755	+
	1756	+ skb_set_end_offset(skb, saved_end_offset);
	1757	+
	1758	+ return 0;
	1759	+}
1564	1760
1565	1761	/**
1566	1762	* skb_copy_expand - copy and expand sk_buff
..	..	@@ -1944,8 +2140,6 @@
1944	2140	struct sk_buff *insp = NULL;
1945	2141
1946	2142	do {
1947		- BUG_ON(!list);
1948		-
1949	2143	if (list->len <= eat) {
1950	2144	/* Eaten as whole. */
1951	2145	eat -= list->len;
..	..	@@ -1953,6 +2147,9 @@
1953	2147	insp = list;
1954	2148	} else {
1955	2149	/* Eaten partially. */
	2150	+ if (skb_is_gso(skb) && !list->head_frag &&
	2151	+ skb_headlen(list))
	2152	+ skb_shinfo(skb)->gso_type \|= SKB_GSO_DODGY;
1956	2153
1957	2154	if (skb_shared(list)) {
1958	2155	/* Sucks! We need to fork list. :-( */
..	..	@@ -1997,10 +2194,12 @@
1997	2194	skb_frag_unref(skb, i);
1998	2195	eat -= size;
1999	2196	} else {
2000		- skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
	2197	+ skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
	2198	+
	2199	+ *frag = skb_shinfo(skb)->frags[i];
2001	2200	if (eat) {
2002		- skb_shinfo(skb)->frags[k].page_offset += eat;
2003		- skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
	2201	+ skb_frag_off_add(frag, eat);
	2202	+ skb_frag_size_sub(frag, eat);
2004	2203	if (!i)
2005	2204	goto end;
2006	2205	eat = 0;
..	..	@@ -2072,7 +2271,7 @@
2072	2271	copy = len;
2073	2272
2074	2273	skb_frag_foreach_page(f,
2075		- f->page_offset + offset - start,
	2274	+ skb_frag_off(f) + offset - start,
2076	2275	copy, p, p_off, p_len, copied) {
2077	2276	vaddr = kmap_atomic(p);
2078	2277	memcpy(to + copied, vaddr + p_off, p_len);
..	..	@@ -2248,7 +2447,7 @@
2248	2447	const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2249	2448
2250	2449	if (__splice_segment(skb_frag_page(f),
2251		- f->page_offset, skb_frag_size(f),
	2450	+ skb_frag_off(f), skb_frag_size(f),
2252	2451	offset, len, spd, false, sk, pipe))
2253	2452	return true;
2254	2453	}
..	..	@@ -2338,20 +2537,20 @@
2338	2537	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2339	2538	skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2340	2539
2341		- if (offset < frag->size)
	2540	+ if (offset < skb_frag_size(frag))
2342	2541	break;
2343	2542
2344		- offset -= frag->size;
	2543	+ offset -= skb_frag_size(frag);
2345	2544	}
2346	2545
2347	2546	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2348	2547	skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2349	2548
2350		- slen = min_t(size_t, len, frag->size - offset);
	2549	+ slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2351	2550
2352	2551	while (slen) {
2353		- ret = kernel_sendpage_locked(sk, frag->page.p,
2354		- frag->page_offset + offset,
	2552	+ ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
	2553	+ skb_frag_off(frag) + offset,
2355	2554	slen, MSG_DONTWAIT);
2356	2555	if (ret <= 0)
2357	2556	goto error;
..	..	@@ -2385,19 +2584,6 @@
2385	2584	return orig_len == len ? ret : orig_len - len;
2386	2585	}
2387	2586	EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2388		-
2389		-/* Send skb data on a socket. */
2390		-int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len)
2391		-{
2392		- int ret = 0;
2393		-
2394		- lock_sock(sk);
2395		- ret = skb_send_sock_locked(sk, skb, offset, len);
2396		- release_sock(sk);
2397		-
2398		- return ret;
2399		-}
2400		-EXPORT_SYMBOL_GPL(skb_send_sock);
2401	2587
2402	2588	/**
2403	2589	* skb_store_bits - store bits from kernel buffer to skb
..	..	@@ -2446,7 +2632,7 @@
2446	2632	copy = len;
2447	2633
2448	2634	skb_frag_foreach_page(frag,
2449		- frag->page_offset + offset - start,
	2635	+ skb_frag_off(frag) + offset - start,
2450	2636	copy, p, p_off, p_len, copied) {
2451	2637	vaddr = kmap_atomic(p);
2452	2638	memcpy(vaddr + p_off, from + copied, p_len);
..	..	@@ -2501,7 +2687,8 @@
2501	2687	if (copy > 0) {
2502	2688	if (copy > len)
2503	2689	copy = len;
2504		- csum = ops->update(skb->data + offset, copy, csum);
	2690	+ csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
	2691	+ skb->data + offset, copy, csum);
2505	2692	if ((len -= copy) == 0)
2506	2693	return csum;
2507	2694	offset += copy;
..	..	@@ -2525,12 +2712,16 @@
2525	2712	copy = len;
2526	2713
2527	2714	skb_frag_foreach_page(frag,
2528		- frag->page_offset + offset - start,
	2715	+ skb_frag_off(frag) + offset - start,
2529	2716	copy, p, p_off, p_len, copied) {
2530	2717	vaddr = kmap_atomic(p);
2531		- csum2 = ops->update(vaddr + p_off, p_len, 0);
	2718	+ csum2 = INDIRECT_CALL_1(ops->update,
	2719	+ csum_partial_ext,
	2720	+ vaddr + p_off, p_len, 0);
2532	2721	kunmap_atomic(vaddr);
2533		- csum = ops->combine(csum, csum2, pos, p_len);
	2722	+ csum = INDIRECT_CALL_1(ops->combine,
	2723	+ csum_block_add_ext, csum,
	2724	+ csum2, pos, p_len);
2534	2725	pos += p_len;
2535	2726	}
2536	2727
..	..	@@ -2553,7 +2744,8 @@
2553	2744	copy = len;
2554	2745	csum2 = __skb_checksum(frag_iter, offset - start,
2555	2746	copy, 0, ops);
2556		- csum = ops->combine(csum, csum2, pos, copy);
	2747	+ csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
	2748	+ csum, csum2, pos, copy);
2557	2749	if ((len -= copy) == 0)
2558	2750	return csum;
2559	2751	offset += copy;
..	..	@@ -2582,19 +2774,20 @@
2582	2774	/* Both of above in one bottle. */
2583	2775
2584	2776	__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2585		- u8 *to, int len, __wsum csum)
	2777	+ u8 *to, int len)
2586	2778	{
2587	2779	int start = skb_headlen(skb);
2588	2780	int i, copy = start - offset;
2589	2781	struct sk_buff *frag_iter;
2590	2782	int pos = 0;
	2783	+ __wsum csum = 0;
2591	2784
2592	2785	/* Copy header. */
2593	2786	if (copy > 0) {
2594	2787	if (copy > len)
2595	2788	copy = len;
2596	2789	csum = csum_partial_copy_nocheck(skb->data + offset, to,
2597		- copy, csum);
	2790	+ copy);
2598	2791	if ((len -= copy) == 0)
2599	2792	return csum;
2600	2793	offset += copy;
..	..	@@ -2619,12 +2812,12 @@
2619	2812	copy = len;
2620	2813
2621	2814	skb_frag_foreach_page(frag,
2622		- frag->page_offset + offset - start,
	2815	+ skb_frag_off(frag) + offset - start,
2623	2816	copy, p, p_off, p_len, copied) {
2624	2817	vaddr = kmap_atomic(p);
2625	2818	csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2626	2819	to + copied,
2627		- p_len, 0);
	2820	+ p_len);
2628	2821	kunmap_atomic(vaddr);
2629	2822	csum = csum_block_add(csum, csum2, pos);
2630	2823	pos += p_len;
..	..	@@ -2650,7 +2843,7 @@
2650	2843	copy = len;
2651	2844	csum2 = skb_copy_and_csum_bits(frag_iter,
2652	2845	offset - start,
2653		- to, copy, 0);
	2846	+ to, copy);
2654	2847	csum = csum_block_add(csum, csum2, pos);
2655	2848	if ((len -= copy) == 0)
2656	2849	return csum;
..	..	@@ -2664,6 +2857,65 @@
2664	2857	return csum;
2665	2858	}
2666	2859	EXPORT_SYMBOL(skb_copy_and_csum_bits);
	2860	+
	2861	+__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
	2862	+{
	2863	+ __sum16 sum;
	2864	+
	2865	+ sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
	2866	+ /* See comments in __skb_checksum_complete(). */
	2867	+ if (likely(!sum)) {
	2868	+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
	2869	+ !skb->csum_complete_sw)
	2870	+ netdev_rx_csum_fault(skb->dev, skb);
	2871	+ }
	2872	+ if (!skb_shared(skb))
	2873	+ skb->csum_valid = !sum;
	2874	+ return sum;
	2875	+}
	2876	+EXPORT_SYMBOL(__skb_checksum_complete_head);
	2877	+
	2878	+/* This function assumes skb->csum already holds pseudo header's checksum,
	2879	+ * which has been changed from the hardware checksum, for example, by
	2880	+ * __skb_checksum_validate_complete(). And, the original skb->csum must
	2881	+ * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
	2882	+ *
	2883	+ * It returns non-zero if the recomputed checksum is still invalid, otherwise
	2884	+ * zero. The new checksum is stored back into skb->csum unless the skb is
	2885	+ * shared.
	2886	+ */
	2887	+__sum16 __skb_checksum_complete(struct sk_buff *skb)
	2888	+{
	2889	+ __wsum csum;
	2890	+ __sum16 sum;
	2891	+
	2892	+ csum = skb_checksum(skb, 0, skb->len, 0);
	2893	+
	2894	+ sum = csum_fold(csum_add(skb->csum, csum));
	2895	+ /* This check is inverted, because we already knew the hardware
	2896	+ * checksum is invalid before calling this function. So, if the
	2897	+ * re-computed checksum is valid instead, then we have a mismatch
	2898	+ * between the original skb->csum and skb_checksum(). This means either
	2899	+ * the original hardware checksum is incorrect or we screw up skb->csum
	2900	+ * when moving skb->data around.
	2901	+ */
	2902	+ if (likely(!sum)) {
	2903	+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
	2904	+ !skb->csum_complete_sw)
	2905	+ netdev_rx_csum_fault(skb->dev, skb);
	2906	+ }
	2907	+
	2908	+ if (!skb_shared(skb)) {
	2909	+ /* Save full packet checksum */
	2910	+ skb->csum = csum;
	2911	+ skb->ip_summed = CHECKSUM_COMPLETE;
	2912	+ skb->csum_complete_sw = 1;
	2913	+ skb->csum_valid = !sum;
	2914	+ }
	2915	+
	2916	+ return sum;
	2917	+}
	2918	+EXPORT_SYMBOL(__skb_checksum_complete);
2667	2919
2668	2920	static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2669	2921	{
..	..	@@ -2779,11 +3031,15 @@
2779	3031	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2780	3032
2781	3033	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
	3034	+ int size;
	3035	+
2782	3036	if (!len)
2783	3037	break;
2784	3038	skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2785		- skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2786		- len -= skb_shinfo(to)->frags[j].size;
	3039	+ size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
	3040	+ len);
	3041	+ skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
	3042	+ len -= size;
2787	3043	skb_frag_ref(to, j);
2788	3044	j++;
2789	3045	}
..	..	@@ -2810,7 +3066,7 @@
2810	3066	csum = 0;
2811	3067	if (csstart != skb->len)
2812	3068	csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2813		- skb->len - csstart, 0);
	3069	+ skb->len - csstart);
2814	3070
2815	3071	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2816	3072	long csstuff = csstart + skb->csum_offset;
..	..	@@ -2985,28 +3241,6 @@
2985	3241	}
2986	3242	EXPORT_SYMBOL(skb_append);
2987	3243
2988		-/**
2989		- * skb_insert - insert a buffer
2990		- * @old: buffer to insert before
2991		- * @newsk: buffer to insert
2992		- * @list: list to use
2993		- *
2994		- * Place a packet before a given packet in a list. The list locks are
2995		- * taken and this function is atomic with respect to other list locked
2996		- * calls.
2997		- *
2998		- * A buffer cannot be placed on two lists at the same time.
2999		- */
3000		-void skb_insert(struct sk_buff old, struct sk_buff newsk, struct sk_buff_head *list)
3001		-{
3002		- unsigned long flags;
3003		-
3004		- spin_lock_irqsave(&list->lock, flags);
3005		- __skb_insert(newsk, old->prev, old, list);
3006		- spin_unlock_irqrestore(&list->lock, flags);
3007		-}
3008		-EXPORT_SYMBOL(skb_insert);
3009		-
3010	3244	static inline void skb_split_inside_header(struct sk_buff *skb,
3011	3245	struct sk_buff* skb1,
3012	3246	const u32 len, const int pos)
..	..	@@ -3056,7 +3290,7 @@
3056	3290	* 2. Split is accurately. We make this.
3057	3291	*/
3058	3292	skb_frag_ref(skb, i);
3059		- skb_shinfo(skb1)->frags[0].page_offset += len - pos;
	3293	+ skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
3060	3294	skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3061	3295	skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3062	3296	skb_shinfo(skb)->nr_frags++;
..	..	@@ -3095,19 +3329,7 @@
3095	3329	*/
3096	3330	static int skb_prepare_for_shift(struct sk_buff *skb)
3097	3331	{
3098		- int ret = 0;
3099		-
3100		- if (skb_cloned(skb)) {
3101		- /* Save and restore truesize: pskb_expand_head() may reallocate
3102		- * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
3103		- * cannot change truesize at this point.
3104		- */
3105		- unsigned int save_truesize = skb->truesize;
3106		-
3107		- ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3108		- skb->truesize = save_truesize;
3109		- }
3110		- return ret;
	3332	+ return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
3111	3333	}
3112	3334
3113	3335	/**
..	..	@@ -3131,7 +3353,7 @@
3131	3353	int skb_shift(struct sk_buff tgt, struct sk_buff skb, int shiftlen)
3132	3354	{
3133	3355	int from, to, merge, todo;
3134		- struct skb_frag_struct fragfrom, fragto;
	3356	+ skb_frag_t fragfrom, fragto;
3135	3357
3136	3358	BUG_ON(shiftlen > skb->len);
3137	3359
..	..	@@ -3150,7 +3372,7 @@
3150	3372	*/
3151	3373	if (!to \|\|
3152	3374	!skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3153		- fragfrom->page_offset)) {
	3375	+ skb_frag_off(fragfrom))) {
3154	3376	merge = -1;
3155	3377	} else {
3156	3378	merge = to - 1;
..	..	@@ -3167,7 +3389,7 @@
3167	3389
3168	3390	skb_frag_size_add(fragto, shiftlen);
3169	3391	skb_frag_size_sub(fragfrom, shiftlen);
3170		- fragfrom->page_offset += shiftlen;
	3392	+ skb_frag_off_add(fragfrom, shiftlen);
3171	3393
3172	3394	goto onlymerged;
3173	3395	}
..	..	@@ -3198,11 +3420,11 @@
3198	3420
3199	3421	} else {
3200	3422	__skb_frag_ref(fragfrom);
3201		- fragto->page = fragfrom->page;
3202		- fragto->page_offset = fragfrom->page_offset;
	3423	+ skb_frag_page_copy(fragto, fragfrom);
	3424	+ skb_frag_off_copy(fragto, fragfrom);
3203	3425	skb_frag_size_set(fragto, todo);
3204	3426
3205		- fragfrom->page_offset += todo;
	3427	+ skb_frag_off_add(fragfrom, todo);
3206	3428	skb_frag_size_sub(fragfrom, todo);
3207	3429	todo = 0;
3208	3430
..	..	@@ -3327,7 +3549,7 @@
3327	3549	if (!st->frag_data)
3328	3550	st->frag_data = kmap_atomic(skb_frag_page(frag));
3329	3551
3330		- data = (u8 ) st->frag_data + frag->page_offset +
	3552	+ data = (u8 ) st->frag_data + skb_frag_off(frag) +
3331	3553	(abs_offset - st->stepped_offset);
3332	3554
3333	3555	return block_limit - abs_offset;
..	..	@@ -3417,64 +3639,6 @@
3417	3639	}
3418	3640	EXPORT_SYMBOL(skb_find_text);
3419	3641
3420		-/**
3421		- * skb_append_datato_frags - append the user data to a skb
3422		- * @sk: sock structure
3423		- * @skb: skb structure to be appended with user data.
3424		- * @getfrag: call back function to be used for getting the user data
3425		- * @from: pointer to user message iov
3426		- * @length: length of the iov message
3427		- *
3428		- * Description: This procedure append the user data in the fragment part
3429		- * of the skb if any page alloc fails user this procedure returns -ENOMEM
3430		- */
3431		-int skb_append_datato_frags(struct sock sk, struct sk_buff skb,
3432		- int (getfrag)(void from, char *to, int offset,
3433		- int len, int odd, struct sk_buff *skb),
3434		- void *from, int length)
3435		-{
3436		- int frg_cnt = skb_shinfo(skb)->nr_frags;
3437		- int copy;
3438		- int offset = 0;
3439		- int ret;
3440		- struct page_frag *pfrag = &current->task_frag;
3441		-
3442		- do {
3443		- /* Return error if we don't have space for new frag */
3444		- if (frg_cnt >= MAX_SKB_FRAGS)
3445		- return -EMSGSIZE;
3446		-
3447		- if (!sk_page_frag_refill(sk, pfrag))
3448		- return -ENOMEM;
3449		-
3450		- /* copy the user data to page */
3451		- copy = min_t(int, length, pfrag->size - pfrag->offset);
3452		-
3453		- ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
3454		- offset, copy, 0, skb);
3455		- if (ret < 0)
3456		- return -EFAULT;
3457		-
3458		- /* copy was successful so update the size parameters */
3459		- skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
3460		- copy);
3461		- frg_cnt++;
3462		- pfrag->offset += copy;
3463		- get_page(pfrag->page);
3464		-
3465		- skb->truesize += copy;
3466		- refcount_add(copy, &sk->sk_wmem_alloc);
3467		- skb->len += copy;
3468		- skb->data_len += copy;
3469		- offset += copy;
3470		- length -= copy;
3471		-
3472		- } while (length > 0);
3473		-
3474		- return 0;
3475		-}
3476		-EXPORT_SYMBOL(skb_append_datato_frags);
3477		-
3478	3642	int skb_append_pagefrags(struct sk_buff skb, struct page page,
3479	3643	int offset, size_t size)
3480	3644	{
..	..	@@ -3521,11 +3685,126 @@
3521	3685	struct page *page;
3522	3686
3523	3687	page = virt_to_head_page(frag_skb->head);
3524		- head_frag.page.p = page;
3525		- head_frag.page_offset = frag_skb->data -
3526		- (unsigned char *)page_address(page);
3527		- head_frag.size = skb_headlen(frag_skb);
	3688	+ __skb_frag_set_page(&head_frag, page);
	3689	+ skb_frag_off_set(&head_frag, frag_skb->data -
	3690	+ (unsigned char *)page_address(page));
	3691	+ skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
3528	3692	return head_frag;
	3693	+}
	3694	+
	3695	+struct sk_buff skb_segment_list(struct sk_buff skb,
	3696	+ netdev_features_t features,
	3697	+ unsigned int offset)
	3698	+{
	3699	+ struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
	3700	+ unsigned int tnl_hlen = skb_tnl_header_len(skb);
	3701	+ unsigned int delta_truesize = 0;
	3702	+ unsigned int delta_len = 0;
	3703	+ struct sk_buff *tail = NULL;
	3704	+ struct sk_buff nskb, tmp;
	3705	+ int len_diff, err;
	3706	+
	3707	+ skb_push(skb, -skb_network_offset(skb) + offset);
	3708	+
	3709	+ /* Ensure the head is writeable before touching the shared info */
	3710	+ err = skb_unclone(skb, GFP_ATOMIC);
	3711	+ if (err)
	3712	+ goto err_linearize;
	3713	+
	3714	+ skb_shinfo(skb)->frag_list = NULL;
	3715	+
	3716	+ while (list_skb) {
	3717	+ nskb = list_skb;
	3718	+ list_skb = list_skb->next;
	3719	+
	3720	+ err = 0;
	3721	+ delta_truesize += nskb->truesize;
	3722	+ if (skb_shared(nskb)) {
	3723	+ tmp = skb_clone(nskb, GFP_ATOMIC);
	3724	+ if (tmp) {
	3725	+ consume_skb(nskb);
	3726	+ nskb = tmp;
	3727	+ err = skb_unclone(nskb, GFP_ATOMIC);
	3728	+ } else {
	3729	+ err = -ENOMEM;
	3730	+ }
	3731	+ }
	3732	+
	3733	+ if (!tail)
	3734	+ skb->next = nskb;
	3735	+ else
	3736	+ tail->next = nskb;
	3737	+
	3738	+ if (unlikely(err)) {
	3739	+ nskb->next = list_skb;
	3740	+ goto err_linearize;
	3741	+ }
	3742	+
	3743	+ tail = nskb;
	3744	+
	3745	+ delta_len += nskb->len;
	3746	+
	3747	+ skb_push(nskb, -skb_network_offset(nskb) + offset);
	3748	+
	3749	+ skb_release_head_state(nskb);
	3750	+ len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
	3751	+ __copy_skb_header(nskb, skb);
	3752	+
	3753	+ skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
	3754	+ nskb->transport_header += len_diff;
	3755	+ skb_copy_from_linear_data_offset(skb, -tnl_hlen,
	3756	+ nskb->data - tnl_hlen,
	3757	+ offset + tnl_hlen);
	3758	+
	3759	+ if (skb_needs_linearize(nskb, features) &&
	3760	+ __skb_linearize(nskb))
	3761	+ goto err_linearize;
	3762	+ }
	3763	+
	3764	+ skb->truesize = skb->truesize - delta_truesize;
	3765	+ skb->data_len = skb->data_len - delta_len;
	3766	+ skb->len = skb->len - delta_len;
	3767	+
	3768	+ skb_gso_reset(skb);
	3769	+
	3770	+ skb->prev = tail;
	3771	+
	3772	+ if (skb_needs_linearize(skb, features) &&
	3773	+ __skb_linearize(skb))
	3774	+ goto err_linearize;
	3775	+
	3776	+ skb_get(skb);
	3777	+
	3778	+ return skb;
	3779	+
	3780	+err_linearize:
	3781	+ kfree_skb_list(skb->next);
	3782	+ skb->next = NULL;
	3783	+ return ERR_PTR(-ENOMEM);
	3784	+}
	3785	+EXPORT_SYMBOL_GPL(skb_segment_list);
	3786	+
	3787	+int skb_gro_receive_list(struct sk_buff p, struct sk_buff skb)
	3788	+{
	3789	+ if (unlikely(p->len + skb->len >= 65536))
	3790	+ return -E2BIG;
	3791	+
	3792	+ if (NAPI_GRO_CB(p)->last == p)
	3793	+ skb_shinfo(p)->frag_list = skb;
	3794	+ else
	3795	+ NAPI_GRO_CB(p)->last->next = skb;
	3796	+
	3797	+ skb_pull(skb, skb_gro_offset(skb));
	3798	+
	3799	+ NAPI_GRO_CB(p)->last = skb;
	3800	+ NAPI_GRO_CB(p)->count++;
	3801	+ p->data_len += skb->len;
	3802	+ p->truesize += skb->truesize;
	3803	+ p->len += skb->len;
	3804	+
	3805	+ NAPI_GRO_CB(skb)->same_flow = 1;
	3806	+
	3807	+ return 0;
3529	3808	}
3530	3809
3531	3810	/**
..	..	@@ -3543,44 +3822,44 @@
3543	3822	struct sk_buff *segs = NULL;
3544	3823	struct sk_buff *tail = NULL;
3545	3824	struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list;
3546		- skb_frag_t *frag = skb_shinfo(head_skb)->frags;
3547	3825	unsigned int mss = skb_shinfo(head_skb)->gso_size;
3548	3826	unsigned int doffset = head_skb->data - skb_mac_header(head_skb);
3549		- struct sk_buff *frag_skb = head_skb;
3550	3827	unsigned int offset = doffset;
3551	3828	unsigned int tnl_hlen = skb_tnl_header_len(head_skb);
3552	3829	unsigned int partial_segs = 0;
3553	3830	unsigned int headroom;
3554	3831	unsigned int len = head_skb->len;
	3832	+ struct sk_buff *frag_skb;
	3833	+ skb_frag_t *frag;
3555	3834	__be16 proto;
3556	3835	bool csum, sg;
3557		- int nfrags = skb_shinfo(head_skb)->nr_frags;
3558	3836	int err = -ENOMEM;
3559	3837	int i = 0;
3560		- int pos;
3561		- int dummy;
	3838	+ int nfrags, pos;
3562	3839
3563		- if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
3564		- (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
3565		- /* gso_size is untrusted, and we have a frag_list with a linear
3566		- * non head_frag head.
3567		- *
3568		- * (we assume checking the first list_skb member suffices;
3569		- * i.e if either of the list_skb members have non head_frag
3570		- * head, then the first one has too).
3571		- *
3572		- * If head_skb's headlen does not fit requested gso_size, it
3573		- * means that the frag_list members do NOT terminate on exact
3574		- * gso_size boundaries. Hence we cannot perform skb_frag_t page
3575		- * sharing. Therefore we must fallback to copying the frag_list
3576		- * skbs; we do so by disabling SG.
3577		- */
3578		- if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
3579		- features &= ~NETIF_F_SG;
	3840	+ if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
	3841	+ mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) {
	3842	+ struct sk_buff *check_skb;
	3843	+
	3844	+ for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
	3845	+ if (skb_headlen(check_skb) && !check_skb->head_frag) {
	3846	+ /* gso_size is untrusted, and we have a frag_list with
	3847	+ * a linear non head_frag item.
	3848	+ *
	3849	+ * If head_skb's headlen does not fit requested gso_size,
	3850	+ * it means that the frag_list members do NOT terminate
	3851	+ * on exact gso_size boundaries. Hence we cannot perform
	3852	+ * skb_frag_t page sharing. Therefore we must fallback to
	3853	+ * copying the frag_list skbs; we do so by disabling SG.
	3854	+ */
	3855	+ features &= ~NETIF_F_SG;
	3856	+ break;
	3857	+ }
	3858	+ }
3580	3859	}
3581	3860
3582	3861	__skb_push(head_skb, doffset);
3583		- proto = skb_network_protocol(head_skb, &dummy);
	3862	+ proto = skb_network_protocol(head_skb, NULL);
3584	3863	if (unlikely(!proto))
3585	3864	return ERR_PTR(-EINVAL);
3586	3865
..	..	@@ -3633,6 +3912,13 @@
3633	3912	headroom = skb_headroom(head_skb);
3634	3913	pos = skb_headlen(head_skb);
3635	3914
	3915	+ if (skb_orphan_frags(head_skb, GFP_ATOMIC))
	3916	+ return ERR_PTR(-ENOMEM);
	3917	+
	3918	+ nfrags = skb_shinfo(head_skb)->nr_frags;
	3919	+ frag = skb_shinfo(head_skb)->frags;
	3920	+ frag_skb = head_skb;
	3921	+
3636	3922	do {
3637	3923	struct sk_buff *nskb;
3638	3924	skb_frag_t *nskb_frag;
..	..	@@ -3657,6 +3943,10 @@
3657	3943	(skb_headlen(list_skb) == len \|\| sg)) {
3658	3944	BUG_ON(skb_headlen(list_skb) > len);
3659	3945
	3946	+ nskb = skb_clone(list_skb, GFP_ATOMIC);
	3947	+ if (unlikely(!nskb))
	3948	+ goto err;
	3949	+
3660	3950	i = 0;
3661	3951	nfrags = skb_shinfo(list_skb)->nr_frags;
3662	3952	frag = skb_shinfo(list_skb)->frags;
..	..	@@ -3675,11 +3965,7 @@
3675	3965	frag++;
3676	3966	}
3677	3967
3678		- nskb = skb_clone(list_skb, GFP_ATOMIC);
3679	3968	list_skb = list_skb->next;
3680		-
3681		- if (unlikely(!nskb))
3682		- goto err;
3683	3969
3684	3970	if (unlikely(pskb_trim(nskb, len))) {
3685	3971	kfree_skb(nskb);
..	..	@@ -3726,14 +4012,20 @@
3726	4012	goto perform_csum_check;
3727	4013
3728	4014	if (!sg) {
3729		- if (!nskb->remcsum_offload)
3730		- nskb->ip_summed = CHECKSUM_NONE;
3731		- SKB_GSO_CB(nskb)->csum =
3732		- skb_copy_and_csum_bits(head_skb, offset,
3733		- skb_put(nskb, len),
3734		- len, 0);
3735		- SKB_GSO_CB(nskb)->csum_start =
3736		- skb_headroom(nskb) + doffset;
	4015	+ if (!csum) {
	4016	+ if (!nskb->remcsum_offload)
	4017	+ nskb->ip_summed = CHECKSUM_NONE;
	4018	+ SKB_GSO_CB(nskb)->csum =
	4019	+ skb_copy_and_csum_bits(head_skb, offset,
	4020	+ skb_put(nskb,
	4021	+ len),
	4022	+ len);
	4023	+ SKB_GSO_CB(nskb)->csum_start =
	4024	+ skb_headroom(nskb) + doffset;
	4025	+ } else {
	4026	+ if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
	4027	+ goto err;
	4028	+ }
3737	4029	continue;
3738	4030	}
3739	4031
..	..	@@ -3745,12 +4037,16 @@
3745	4037	skb_shinfo(nskb)->tx_flags \|= skb_shinfo(head_skb)->tx_flags &
3746	4038	SKBTX_SHARED_FRAG;
3747	4039
3748		- if (skb_orphan_frags(frag_skb, GFP_ATOMIC) \|\|
3749		- skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
	4040	+ if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC))
3750	4041	goto err;
3751	4042
3752	4043	while (pos < offset + len) {
3753	4044	if (i >= nfrags) {
	4045	+ if (skb_orphan_frags(list_skb, GFP_ATOMIC) \|\|
	4046	+ skb_zerocopy_clone(nskb, list_skb,
	4047	+ GFP_ATOMIC))
	4048	+ goto err;
	4049	+
3754	4050	i = 0;
3755	4051	nfrags = skb_shinfo(list_skb)->nr_frags;
3756	4052	frag = skb_shinfo(list_skb)->frags;
..	..	@@ -3764,10 +4060,6 @@
3764	4060	i--;
3765	4061	frag--;
3766	4062	}
3767		- if (skb_orphan_frags(frag_skb, GFP_ATOMIC) \|\|
3768		- skb_zerocopy_clone(nskb, frag_skb,
3769		- GFP_ATOMIC))
3770		- goto err;
3771	4063
3772	4064	list_skb = list_skb->next;
3773	4065	}
..	..	@@ -3786,7 +4078,7 @@
3786	4078	size = skb_frag_size(nskb_frag);
3787	4079
3788	4080	if (pos < offset) {
3789		- nskb_frag->page_offset += offset - pos;
	4081	+ skb_frag_off_add(nskb_frag, offset - pos);
3790	4082	skb_frag_size_sub(nskb_frag, offset - pos);
3791	4083	}
3792	4084
..	..	@@ -3907,7 +4199,7 @@
3907	4199	--frag = --frag2;
3908	4200	} while (--i);
3909	4201
3910		- frag->page_offset += offset;
	4202	+ skb_frag_off_add(frag, offset);
3911	4203	skb_frag_size_sub(frag, offset);
3912	4204
3913	4205	/* all fragments truesize : remove (head size + sk_buff) */
..	..	@@ -3936,8 +4228,8 @@
3936	4228
3937	4229	pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3938	4230
3939		- frag->page.p = page;
3940		- frag->page_offset = first_offset;
	4231	+ __skb_frag_set_page(frag, page);
	4232	+ skb_frag_off_set(frag, first_offset);
3941	4233	skb_frag_size_set(frag, first_size);
3942	4234
3943	4235	memcpy(frag + 1, skbinfo->frags, sizeof(frag) skbinfo->nr_frags);
..	..	@@ -3953,7 +4245,7 @@
3953	4245	if (offset > headlen) {
3954	4246	unsigned int eat = offset - headlen;
3955	4247
3956		- skbinfo->frags[0].page_offset += eat;
	4248	+ skb_frag_off_add(&skbinfo->frags[0], eat);
3957	4249	skb_frag_size_sub(&skbinfo->frags[0], eat);
3958	4250	skb->data_len -= eat;
3959	4251	skb->len -= eat;
..	..	@@ -3983,7 +4275,58 @@
3983	4275	NAPI_GRO_CB(skb)->same_flow = 1;
3984	4276	return 0;
3985	4277	}
3986		-EXPORT_SYMBOL_GPL(skb_gro_receive);
	4278	+
	4279	+#ifdef CONFIG_SKB_EXTENSIONS
	4280	+#define SKB_EXT_ALIGN_VALUE 8
	4281	+#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
	4282	+
	4283	+static const u8 skb_ext_type_len[] = {
	4284	+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	4285	+ [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
	4286	+#endif
	4287	+#ifdef CONFIG_XFRM
	4288	+ [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
	4289	+#endif
	4290	+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	4291	+ [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
	4292	+#endif
	4293	+#if IS_ENABLED(CONFIG_MPTCP)
	4294	+ [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
	4295	+#endif
	4296	+};
	4297	+
	4298	+static __always_inline unsigned int skb_ext_total_length(void)
	4299	+{
	4300	+ return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
	4301	+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	4302	+ skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
	4303	+#endif
	4304	+#ifdef CONFIG_XFRM
	4305	+ skb_ext_type_len[SKB_EXT_SEC_PATH] +
	4306	+#endif
	4307	+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	4308	+ skb_ext_type_len[TC_SKB_EXT] +
	4309	+#endif
	4310	+#if IS_ENABLED(CONFIG_MPTCP)
	4311	+ skb_ext_type_len[SKB_EXT_MPTCP] +
	4312	+#endif
	4313	+ 0;
	4314	+}
	4315	+
	4316	+static void skb_extensions_init(void)
	4317	+{
	4318	+ BUILD_BUG_ON(SKB_EXT_NUM >= 8);
	4319	+ BUILD_BUG_ON(skb_ext_total_length() > 255);
	4320	+
	4321	+ skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
	4322	+ SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
	4323	+ 0,
	4324	+ SLAB_HWCACHE_ALIGN\|SLAB_PANIC,
	4325	+ NULL);
	4326	+}
	4327	+#else
	4328	+static void skb_extensions_init(void) {}
	4329	+#endif
3987	4330
3988	4331	void __init skb_init(void)
3989	4332	{
..	..	@@ -3999,6 +4342,7 @@
3999	4342	0,
4000	4343	SLAB_HWCACHE_ALIGN\|SLAB_PANIC,
4001	4344	NULL);
	4345	+ skb_extensions_init();
4002	4346	}
4003	4347
4004	4348	static int
..	..	@@ -4037,7 +4381,7 @@
4037	4381	if (copy > len)
4038	4382	copy = len;
4039	4383	sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4040		- frag->page_offset+offset-start);
	4384	+ skb_frag_off(frag) + offset - start);
4041	4385	elt++;
4042	4386	if (!(len -= copy))
4043	4387	return elt;
..	..	@@ -4154,7 +4498,7 @@
4154	4498	* at the moment even if they are anonymous).
4155	4499	*/
4156	4500	if ((skb_cloned(skb) \|\| skb_shinfo(skb)->nr_frags) &&
4157		- __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
	4501	+ !__pskb_pull_tail(skb, __skb_pagelen(skb)))
4158	4502	return -ENOMEM;
4159	4503
4160	4504	/* Easy case. Most of packets will go this way. */
..	..	@@ -4258,7 +4602,7 @@
4258	4602	int sock_queue_err_skb(struct sock sk, struct sk_buff skb)
4259	4603	{
4260	4604	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4261		- (unsigned int)sk->sk_rcvbuf)
	4605	+ (unsigned int)READ_ONCE(sk->sk_rcvbuf))
4262	4606	return -ENOMEM;
4263	4607
4264	4608	skb_orphan(skb);
..	..	@@ -4377,7 +4721,7 @@
4377	4721	{
4378	4722	bool ret;
4379	4723
4380		- if (likely(sysctl_tstamp_allow_data \|\| tsonly))
	4724	+ if (likely(READ_ONCE(sysctl_tstamp_allow_data) \|\| tsonly))
4381	4725	return true;
4382	4726
4383	4727	read_lock_bh(&sk->sk_callback_lock);
..	..	@@ -4433,13 +4777,18 @@
4433	4777	if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4434	4778	sk->sk_protocol == IPPROTO_TCP &&
4435	4779	sk->sk_type == SOCK_STREAM) {
4436		- skb = tcp_get_timestamping_opt_stats(sk);
	4780	+ skb = tcp_get_timestamping_opt_stats(sk, orig_skb);
4437	4781	opt_stats = true;
4438	4782	} else
4439	4783	#endif
4440	4784	skb = alloc_skb(0, GFP_ATOMIC);
4441	4785	} else {
4442	4786	skb = skb_clone(orig_skb, GFP_ATOMIC);
	4787	+
	4788	+ if (skb_orphan_frags_rx(skb, GFP_ATOMIC)) {
	4789	+ kfree_skb(skb);
	4790	+ return;
	4791	+ }
4443	4792	}
4444	4793	if (!skb)
4445	4794	return;
..	..	@@ -4550,9 +4899,9 @@
4550	4899	typeof(IPPROTO_IP) proto,
4551	4900	unsigned int off)
4552	4901	{
4553		- switch (proto) {
4554		- int err;
	4902	+ int err;
4555	4903
	4904	+ switch (proto) {
4556	4905	case IPPROTO_TCP:
4557	4906	err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4558	4907	off + MAX_TCP_HDR_LEN);
..	..	@@ -4595,7 +4944,7 @@
4595	4944	if (err < 0)
4596	4945	goto out;
4597	4946
4598		- if (ip_hdr(skb)->frag_off & htons(IP_OFFSET \| IP_MF))
	4947	+ if (ip_is_fragment(ip_hdr(skb)))
4599	4948	fragment = true;
4600	4949
4601	4950	off = ip_hdrlen(skb);
..	..	@@ -4962,13 +5311,13 @@
4962	5311	skb->skb_iif = 0;
4963	5312	skb->ignore_df = 0;
4964	5313	skb_dst_drop(skb);
4965		- secpath_reset(skb);
4966		- nf_reset(skb);
	5314	+ skb_ext_reset(skb);
	5315	+ nf_reset_ct(skb);
4967	5316	nf_reset_trace(skb);
4968	5317
4969	5318	#ifdef CONFIG_NET_SWITCHDEV
4970	5319	skb->offload_fwd_mark = 0;
4971		- skb->offload_mr_fwd_mark = 0;
	5320	+ skb->offload_l3_fwd_mark = 0;
4972	5321	#endif
4973	5322
4974	5323	if (!xnet)
..	..	@@ -5060,6 +5409,8 @@
5060	5409	* - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
5061	5410	*
5062	5411	* This is a helper to do that correctly considering GSO_BY_FRAGS.
	5412	+ *
	5413	+ * @skb: GSO skb
5063	5414	*
5064	5415	* @seg_len: The segmented length (from skb_gso_*_seglen). In the
5065	5416	* GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
..	..	@@ -5246,7 +5597,7 @@
5246	5597	int err;
5247	5598
5248	5599	if (likely(skb_vlan_tag_present(skb))) {
5249		- skb->vlan_tci = 0;
	5600	+ __vlan_hwaccel_clear_tag(skb);
5250	5601	} else {
5251	5602	if (unlikely(!eth_type_vlan(skb->protocol)))
5252	5603	return 0;
..	..	@@ -5298,6 +5649,252 @@
5298	5649	return 0;
5299	5650	}
5300	5651	EXPORT_SYMBOL(skb_vlan_push);
	5652	+
	5653	+/**
	5654	+ * skb_eth_pop() - Drop the Ethernet header at the head of a packet
	5655	+ *
	5656	+ * @skb: Socket buffer to modify
	5657	+ *
	5658	+ * Drop the Ethernet header of @skb.
	5659	+ *
	5660	+ * Expects that skb->data points to the mac header and that no VLAN tags are
	5661	+ * present.
	5662	+ *
	5663	+ * Returns 0 on success, -errno otherwise.
	5664	+ */
	5665	+int skb_eth_pop(struct sk_buff *skb)
	5666	+{
	5667	+ if (!pskb_may_pull(skb, ETH_HLEN) \|\| skb_vlan_tagged(skb) \|\|
	5668	+ skb_network_offset(skb) < ETH_HLEN)
	5669	+ return -EPROTO;
	5670	+
	5671	+ skb_pull_rcsum(skb, ETH_HLEN);
	5672	+ skb_reset_mac_header(skb);
	5673	+ skb_reset_mac_len(skb);
	5674	+
	5675	+ return 0;
	5676	+}
	5677	+EXPORT_SYMBOL(skb_eth_pop);
	5678	+
	5679	+/**
	5680	+ * skb_eth_push() - Add a new Ethernet header at the head of a packet
	5681	+ *
	5682	+ * @skb: Socket buffer to modify
	5683	+ * @dst: Destination MAC address of the new header
	5684	+ * @src: Source MAC address of the new header
	5685	+ *
	5686	+ * Prepend @skb with a new Ethernet header.
	5687	+ *
	5688	+ * Expects that skb->data points to the mac header, which must be empty.
	5689	+ *
	5690	+ * Returns 0 on success, -errno otherwise.
	5691	+ */
	5692	+int skb_eth_push(struct sk_buff skb, const unsigned char dst,
	5693	+ const unsigned char *src)
	5694	+{
	5695	+ struct ethhdr *eth;
	5696	+ int err;
	5697	+
	5698	+ if (skb_network_offset(skb) \|\| skb_vlan_tag_present(skb))
	5699	+ return -EPROTO;
	5700	+
	5701	+ err = skb_cow_head(skb, sizeof(*eth));
	5702	+ if (err < 0)
	5703	+ return err;
	5704	+
	5705	+ skb_push(skb, sizeof(*eth));
	5706	+ skb_reset_mac_header(skb);
	5707	+ skb_reset_mac_len(skb);
	5708	+
	5709	+ eth = eth_hdr(skb);
	5710	+ ether_addr_copy(eth->h_dest, dst);
	5711	+ ether_addr_copy(eth->h_source, src);
	5712	+ eth->h_proto = skb->protocol;
	5713	+
	5714	+ skb_postpush_rcsum(skb, eth, sizeof(*eth));
	5715	+
	5716	+ return 0;
	5717	+}
	5718	+EXPORT_SYMBOL(skb_eth_push);
	5719	+
	5720	+/* Update the ethertype of hdr and the skb csum value if required. */
	5721	+static void skb_mod_eth_type(struct sk_buff skb, struct ethhdr hdr,
	5722	+ __be16 ethertype)
	5723	+{
	5724	+ if (skb->ip_summed == CHECKSUM_COMPLETE) {
	5725	+ __be16 diff[] = { ~hdr->h_proto, ethertype };
	5726	+
	5727	+ skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	5728	+ }
	5729	+
	5730	+ hdr->h_proto = ethertype;
	5731	+}
	5732	+
	5733	+/**
	5734	+ * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
	5735	+ * the packet
	5736	+ *
	5737	+ * @skb: buffer
	5738	+ * @mpls_lse: MPLS label stack entry to push
	5739	+ * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
	5740	+ * @mac_len: length of the MAC header
	5741	+ * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
	5742	+ * ethernet
	5743	+ *
	5744	+ * Expects skb->data at mac header.
	5745	+ *
	5746	+ * Returns 0 on success, -errno otherwise.
	5747	+ */
	5748	+int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
	5749	+ int mac_len, bool ethernet)
	5750	+{
	5751	+ struct mpls_shim_hdr *lse;
	5752	+ int err;
	5753	+
	5754	+ if (unlikely(!eth_p_mpls(mpls_proto)))
	5755	+ return -EINVAL;
	5756	+
	5757	+ /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
	5758	+ if (skb->encapsulation)
	5759	+ return -EINVAL;
	5760	+
	5761	+ err = skb_cow_head(skb, MPLS_HLEN);
	5762	+ if (unlikely(err))
	5763	+ return err;
	5764	+
	5765	+ if (!skb->inner_protocol) {
	5766	+ skb_set_inner_network_header(skb, skb_network_offset(skb));
	5767	+ skb_set_inner_protocol(skb, skb->protocol);
	5768	+ }
	5769	+
	5770	+ skb_push(skb, MPLS_HLEN);
	5771	+ memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
	5772	+ mac_len);
	5773	+ skb_reset_mac_header(skb);
	5774	+ skb_set_network_header(skb, mac_len);
	5775	+ skb_reset_mac_len(skb);
	5776	+
	5777	+ lse = mpls_hdr(skb);
	5778	+ lse->label_stack_entry = mpls_lse;
	5779	+ skb_postpush_rcsum(skb, lse, MPLS_HLEN);
	5780	+
	5781	+ if (ethernet && mac_len >= ETH_HLEN)
	5782	+ skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
	5783	+ skb->protocol = mpls_proto;
	5784	+
	5785	+ return 0;
	5786	+}
	5787	+EXPORT_SYMBOL_GPL(skb_mpls_push);
	5788	+
	5789	+/**
	5790	+ * skb_mpls_pop() - pop the outermost MPLS header
	5791	+ *
	5792	+ * @skb: buffer
	5793	+ * @next_proto: ethertype of header after popped MPLS header
	5794	+ * @mac_len: length of the MAC header
	5795	+ * @ethernet: flag to indicate if the packet is ethernet
	5796	+ *
	5797	+ * Expects skb->data at mac header.
	5798	+ *
	5799	+ * Returns 0 on success, -errno otherwise.
	5800	+ */
	5801	+int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
	5802	+ bool ethernet)
	5803	+{
	5804	+ int err;
	5805	+
	5806	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5807	+ return 0;
	5808	+
	5809	+ err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
	5810	+ if (unlikely(err))
	5811	+ return err;
	5812	+
	5813	+ skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
	5814	+ memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
	5815	+ mac_len);
	5816	+
	5817	+ __skb_pull(skb, MPLS_HLEN);
	5818	+ skb_reset_mac_header(skb);
	5819	+ skb_set_network_header(skb, mac_len);
	5820	+
	5821	+ if (ethernet && mac_len >= ETH_HLEN) {
	5822	+ struct ethhdr *hdr;
	5823	+
	5824	+ /* use mpls_hdr() to get ethertype to account for VLANs. */
	5825	+ hdr = (struct ethhdr )((void )mpls_hdr(skb) - ETH_HLEN);
	5826	+ skb_mod_eth_type(skb, hdr, next_proto);
	5827	+ }
	5828	+ skb->protocol = next_proto;
	5829	+
	5830	+ return 0;
	5831	+}
	5832	+EXPORT_SYMBOL_GPL(skb_mpls_pop);
	5833	+
	5834	+/**
	5835	+ * skb_mpls_update_lse() - modify outermost MPLS header and update csum
	5836	+ *
	5837	+ * @skb: buffer
	5838	+ * @mpls_lse: new MPLS label stack entry to update to
	5839	+ *
	5840	+ * Expects skb->data at mac header.
	5841	+ *
	5842	+ * Returns 0 on success, -errno otherwise.
	5843	+ */
	5844	+int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
	5845	+{
	5846	+ int err;
	5847	+
	5848	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5849	+ return -EINVAL;
	5850	+
	5851	+ err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
	5852	+ if (unlikely(err))
	5853	+ return err;
	5854	+
	5855	+ if (skb->ip_summed == CHECKSUM_COMPLETE) {
	5856	+ __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
	5857	+
	5858	+ skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	5859	+ }
	5860	+
	5861	+ mpls_hdr(skb)->label_stack_entry = mpls_lse;
	5862	+
	5863	+ return 0;
	5864	+}
	5865	+EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
	5866	+
	5867	+/**
	5868	+ * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
	5869	+ *
	5870	+ * @skb: buffer
	5871	+ *
	5872	+ * Expects skb->data at mac header.
	5873	+ *
	5874	+ * Returns 0 on success, -errno otherwise.
	5875	+ */
	5876	+int skb_mpls_dec_ttl(struct sk_buff *skb)
	5877	+{
	5878	+ u32 lse;
	5879	+ u8 ttl;
	5880	+
	5881	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5882	+ return -EINVAL;
	5883	+
	5884	+ if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
	5885	+ return -ENOMEM;
	5886	+
	5887	+ lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
	5888	+ ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
	5889	+ if (!--ttl)
	5890	+ return -EINVAL;
	5891	+
	5892	+ lse &= ~MPLS_LS_TTL_MASK;
	5893	+ lse \|= ttl << MPLS_LS_TTL_SHIFT;
	5894	+
	5895	+ return skb_mpls_update_lse(skb, cpu_to_be32(lse));
	5896	+}
	5897	+EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
5301	5898
5302	5899	/**
5303	5900	* alloc_skb_with_frags - allocate skb with page frags
..	..	@@ -5421,11 +6018,7 @@
5421	6018	skb->head = data;
5422	6019	skb->data = data;
5423	6020	skb->head_frag = 0;
5424		-#ifdef NET_SKBUFF_DATA_USES_OFFSET
5425		- skb->end = size;
5426		-#else
5427		- skb->end = skb->head + size;
5428		-#endif
	6021	+ skb_set_end_offset(skb, size);
5429	6022	skb_set_tail_pointer(skb, skb_headlen(skb));
5430	6023	skb_headers_offset_update(skb, 0);
5431	6024	skb->cloned = 0;
..	..	@@ -5517,8 +6110,7 @@
5517	6110	size = SKB_WITH_OVERHEAD(ksize(data));
5518	6111
5519	6112	memcpy((struct skb_shared_info *)(data + size),
5520		- skb_shinfo(skb), offsetof(struct skb_shared_info,
5521		- frags[skb_shinfo(skb)->nr_frags]));
	6113	+ skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
5522	6114	if (skb_orphan_frags(skb, gfp_mask)) {
5523	6115	kfree(data);
5524	6116	return -ENOMEM;
..	..	@@ -5539,7 +6131,7 @@
5539	6131	* where splitting is expensive.
5540	6132	* 2. Split is accurately. We make this.
5541	6133	*/
5542		- shinfo->frags[0].page_offset += off - pos;
	6134	+ skb_frag_off_add(&shinfo->frags[0], off - pos);
5543	6135	skb_frag_size_sub(&shinfo->frags[0], off - pos);
5544	6136	}
5545	6137	skb_frag_ref(skb, i);
..	..	@@ -5564,11 +6156,7 @@
5564	6156	skb->head = data;
5565	6157	skb->head_frag = 0;
5566	6158	skb->data = data;
5567		-#ifdef NET_SKBUFF_DATA_USES_OFFSET
5568		- skb->end = size;
5569		-#else
5570		- skb->end = skb->head + size;
5571		-#endif
	6159	+ skb_set_end_offset(skb, size);
5572	6160	skb_reset_tail_pointer(skb);
5573	6161	skb_headers_offset_update(skb, 0);
5574	6162	skb->cloned = 0;
..	..	@@ -5642,4 +6230,181 @@
5642	6230	*/
5643	6231	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5644	6232	}
5645		-EXPORT_SYMBOL_GPL(skb_condense);
	6233	+
	6234	+#ifdef CONFIG_SKB_EXTENSIONS
	6235	+static void skb_ext_get_ptr(struct skb_ext ext, enum skb_ext_id id)
	6236	+{
	6237	+ return (void )ext + (ext->offset[id] SKB_EXT_ALIGN_VALUE);
	6238	+}
	6239	+
	6240	+/**
	6241	+ * __skb_ext_alloc - allocate a new skb extensions storage
	6242	+ *
	6243	+ * @flags: See kmalloc().
	6244	+ *
	6245	+ * Returns the newly allocated pointer. The pointer can later attached to a
	6246	+ * skb via __skb_ext_set().
	6247	+ * Note: caller must handle the skb_ext as an opaque data.
	6248	+ */
	6249	+struct skb_ext *__skb_ext_alloc(gfp_t flags)
	6250	+{
	6251	+ struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
	6252	+
	6253	+ if (new) {
	6254	+ memset(new->offset, 0, sizeof(new->offset));
	6255	+ refcount_set(&new->refcnt, 1);
	6256	+ }
	6257	+
	6258	+ return new;
	6259	+}
	6260	+
	6261	+static struct skb_ext skb_ext_maybe_cow(struct skb_ext old,
	6262	+ unsigned int old_active)
	6263	+{
	6264	+ struct skb_ext *new;
	6265	+
	6266	+ if (refcount_read(&old->refcnt) == 1)
	6267	+ return old;
	6268	+
	6269	+ new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
	6270	+ if (!new)
	6271	+ return NULL;
	6272	+
	6273	+ memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
	6274	+ refcount_set(&new->refcnt, 1);
	6275	+
	6276	+#ifdef CONFIG_XFRM
	6277	+ if (old_active & (1 << SKB_EXT_SEC_PATH)) {
	6278	+ struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
	6279	+ unsigned int i;
	6280	+
	6281	+ for (i = 0; i < sp->len; i++)
	6282	+ xfrm_state_hold(sp->xvec[i]);
	6283	+ }
	6284	+#endif
	6285	+ __skb_ext_put(old);
	6286	+ return new;
	6287	+}
	6288	+
	6289	+/**
	6290	+ * __skb_ext_set - attach the specified extension storage to this skb
	6291	+ * @skb: buffer
	6292	+ * @id: extension id
	6293	+ * @ext: extension storage previously allocated via __skb_ext_alloc()
	6294	+ *
	6295	+ * Existing extensions, if any, are cleared.
	6296	+ *
	6297	+ * Returns the pointer to the extension.
	6298	+ */
	6299	+void __skb_ext_set(struct sk_buff skb, enum skb_ext_id id,
	6300	+ struct skb_ext *ext)
	6301	+{
	6302	+ unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
	6303	+
	6304	+ skb_ext_put(skb);
	6305	+ newlen = newoff + skb_ext_type_len[id];
	6306	+ ext->chunks = newlen;
	6307	+ ext->offset[id] = newoff;
	6308	+ skb->extensions = ext;
	6309	+ skb->active_extensions = 1 << id;
	6310	+ return skb_ext_get_ptr(ext, id);
	6311	+}
	6312	+
	6313	+/**
	6314	+ * skb_ext_add - allocate space for given extension, COW if needed
	6315	+ * @skb: buffer
	6316	+ * @id: extension to allocate space for
	6317	+ *
	6318	+ * Allocates enough space for the given extension.
	6319	+ * If the extension is already present, a pointer to that extension
	6320	+ * is returned.
	6321	+ *
	6322	+ * If the skb was cloned, COW applies and the returned memory can be
	6323	+ * modified without changing the extension space of clones buffers.
	6324	+ *
	6325	+ * Returns pointer to the extension or NULL on allocation failure.
	6326	+ */
	6327	+void skb_ext_add(struct sk_buff skb, enum skb_ext_id id)
	6328	+{
	6329	+ struct skb_ext new, old = NULL;
	6330	+ unsigned int newlen, newoff;
	6331	+
	6332	+ if (skb->active_extensions) {
	6333	+ old = skb->extensions;
	6334	+
	6335	+ new = skb_ext_maybe_cow(old, skb->active_extensions);
	6336	+ if (!new)
	6337	+ return NULL;
	6338	+
	6339	+ if (__skb_ext_exist(new, id))
	6340	+ goto set_active;
	6341	+
	6342	+ newoff = new->chunks;
	6343	+ } else {
	6344	+ newoff = SKB_EXT_CHUNKSIZEOF(*new);
	6345	+
	6346	+ new = __skb_ext_alloc(GFP_ATOMIC);
	6347	+ if (!new)
	6348	+ return NULL;
	6349	+ }
	6350	+
	6351	+ newlen = newoff + skb_ext_type_len[id];
	6352	+ new->chunks = newlen;
	6353	+ new->offset[id] = newoff;
	6354	+set_active:
	6355	+ skb->extensions = new;
	6356	+ skb->active_extensions \|= 1 << id;
	6357	+ return skb_ext_get_ptr(new, id);
	6358	+}
	6359	+EXPORT_SYMBOL(skb_ext_add);
	6360	+
	6361	+#ifdef CONFIG_XFRM
	6362	+static void skb_ext_put_sp(struct sec_path *sp)
	6363	+{
	6364	+ unsigned int i;
	6365	+
	6366	+ for (i = 0; i < sp->len; i++)
	6367	+ xfrm_state_put(sp->xvec[i]);
	6368	+}
	6369	+#endif
	6370	+
	6371	+void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
	6372	+{
	6373	+ struct skb_ext *ext = skb->extensions;
	6374	+
	6375	+ skb->active_extensions &= ~(1 << id);
	6376	+ if (skb->active_extensions == 0) {
	6377	+ skb->extensions = NULL;
	6378	+ __skb_ext_put(ext);
	6379	+#ifdef CONFIG_XFRM
	6380	+ } else if (id == SKB_EXT_SEC_PATH &&
	6381	+ refcount_read(&ext->refcnt) == 1) {
	6382	+ struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
	6383	+
	6384	+ skb_ext_put_sp(sp);
	6385	+ sp->len = 0;
	6386	+#endif
	6387	+ }
	6388	+}
	6389	+EXPORT_SYMBOL(__skb_ext_del);
	6390	+
	6391	+void __skb_ext_put(struct skb_ext *ext)
	6392	+{
	6393	+ /* If this is last clone, nothing can increment
	6394	+ * it after check passes. Avoids one atomic op.
	6395	+ */
	6396	+ if (refcount_read(&ext->refcnt) == 1)
	6397	+ goto free_now;
	6398	+
	6399	+ if (!refcount_dec_and_test(&ext->refcnt))
	6400	+ return;
	6401	+free_now:
	6402	+#ifdef CONFIG_XFRM
	6403	+ if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
	6404	+ skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
	6405	+#endif
	6406	+
	6407	+ kmem_cache_free(skbuff_ext_cache, ext);
	6408	+}
	6409	+EXPORT_SYMBOL(__skb_ext_put);
	6410	+#endif /* CONFIG_SKB_EXTENSIONS */