~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* Routines having to do with the 'struct sk_buff' memory handlers.
3	4	*
..	..	@@ -25,11 +26,6 @@
25	26	* disabled, or you better be real sure that the operation is atomic
26	27	* with respect to whatever list is being frobbed (e.g. via lock_sock()
27	28	* or via disabling bottom half handlers, etc).
28		- *
29		- * This program is free software; you can redistribute it and/or
30		- * modify it under the terms of the GNU General Public License
31		- * as published by the Free Software Foundation; either version
32		- * 2 of the License, or (at your option) any later version.
33	29	*/
34	30
35	31	/*
..	..	@@ -63,7 +59,7 @@
63	59	#include <linux/errqueue.h>
64	60	#include <linux/prefetch.h>
65	61	#include <linux/if_vlan.h>
66		-#include <linux/locallock.h>
	62	+#include <linux/mpls.h>
67	63
68	64	#include <net/protocol.h>
69	65	#include <net/dst.h>
..	..	@@ -71,15 +67,24 @@
71	67	#include <net/checksum.h>
72	68	#include <net/ip6_checksum.h>
73	69	#include <net/xfrm.h>
	70	+#include <net/mpls.h>
	71	+#include <net/mptcp.h>
74	72
75	73	#include <linux/uaccess.h>
76	74	#include <trace/events/skb.h>
77	75	#include <linux/highmem.h>
78	76	#include <linux/capability.h>
79	77	#include <linux/user_namespace.h>
	78	+#include <linux/indirect_call_wrapper.h>
	79	+#include <trace/hooks/net.h>
	80	+
	81	+#include "datagram.h"
80	82
81	83	struct kmem_cache *skbuff_head_cache __ro_after_init;
82	84	static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
	85	+#ifdef CONFIG_SKB_EXTENSIONS
	86	+static struct kmem_cache *skbuff_ext_cache __ro_after_init;
	87	+#endif
83	88	int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
84	89	EXPORT_SYMBOL(sysctl_max_skb_frags);
85	90
..	..	@@ -98,7 +103,7 @@
98	103	static void skb_panic(struct sk_buff skb, unsigned int sz, void addr,
99	104	const char msg[])
100	105	{
101		- pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n",
	106	+ pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n",
102	107	msg, addr, skb->len, sz, skb->head, skb->data,
103	108	(unsigned long)skb->tail, (unsigned long)skb->end,
104	109	skb->dev ? skb->dev->name : "<NULL>");
..	..	@@ -245,6 +250,9 @@
245	250
246	251	fclones->skb2.fclone = SKB_FCLONE_CLONE;
247	252	}
	253	+
	254	+ skb_set_kcov_handle(skb, kcov_common_handle());
	255	+
248	256	out:
249	257	return skb;
250	258	nodata:
..	..	@@ -253,6 +261,35 @@
253	261	goto out;
254	262	}
255	263	EXPORT_SYMBOL(__alloc_skb);
	264	+
	265	+/* Caller must provide SKB that is memset cleared */
	266	+static struct sk_buff __build_skb_around(struct sk_buff skb,
	267	+ void *data, unsigned int frag_size)
	268	+{
	269	+ struct skb_shared_info *shinfo;
	270	+ unsigned int size = frag_size ? : ksize(data);
	271	+
	272	+ size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
	273	+
	274	+ /* Assumes caller memset cleared SKB */
	275	+ skb->truesize = SKB_TRUESIZE(size);
	276	+ refcount_set(&skb->users, 1);
	277	+ skb->head = data;
	278	+ skb->data = data;
	279	+ skb_reset_tail_pointer(skb);
	280	+ skb->end = skb->tail + size;
	281	+ skb->mac_header = (typeof(skb->mac_header))~0U;
	282	+ skb->transport_header = (typeof(skb->transport_header))~0U;
	283	+
	284	+ /* make sure we initialize shinfo sequentially */
	285	+ shinfo = skb_shinfo(skb);
	286	+ memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
	287	+ atomic_set(&shinfo->dataref, 1);
	288	+
	289	+ skb_set_kcov_handle(skb, kcov_common_handle());
	290	+
	291	+ return skb;
	292	+}
256	293
257	294	/**
258	295	* __build_skb - build a network buffer
..	..	@@ -275,32 +312,15 @@
275	312	*/
276	313	struct sk_buff __build_skb(void data, unsigned int frag_size)
277	314	{
278		- struct skb_shared_info *shinfo;
279	315	struct sk_buff *skb;
280		- unsigned int size = frag_size ? : ksize(data);
281	316
282	317	skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC);
283		- if (!skb)
	318	+ if (unlikely(!skb))
284	319	return NULL;
285	320
286		- size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
287		-
288	321	memset(skb, 0, offsetof(struct sk_buff, tail));
289		- skb->truesize = SKB_TRUESIZE(size);
290		- refcount_set(&skb->users, 1);
291		- skb->head = data;
292		- skb->data = data;
293		- skb_reset_tail_pointer(skb);
294		- skb->end = skb->tail + size;
295		- skb->mac_header = (typeof(skb->mac_header))~0U;
296		- skb->transport_header = (typeof(skb->transport_header))~0U;
297	322
298		- /* make sure we initialize shinfo sequentially */
299		- shinfo = skb_shinfo(skb);
300		- memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
301		- atomic_set(&shinfo->dataref, 1);
302		-
303		- return skb;
	323	+ return __build_skb_around(skb, data, frag_size);
304	324	}
305	325
306	326	/* build_skb() is wrapper over __build_skb(), that specifically
..	..	@@ -321,6 +341,29 @@
321	341	}
322	342	EXPORT_SYMBOL(build_skb);
323	343
	344	+/**
	345	+ * build_skb_around - build a network buffer around provided skb
	346	+ * @skb: sk_buff provide by caller, must be memset cleared
	347	+ * @data: data buffer provided by caller
	348	+ * @frag_size: size of data, or 0 if head was kmalloced
	349	+ */
	350	+struct sk_buff build_skb_around(struct sk_buff skb,
	351	+ void *data, unsigned int frag_size)
	352	+{
	353	+ if (unlikely(!skb))
	354	+ return NULL;
	355	+
	356	+ skb = __build_skb_around(skb, data, frag_size);
	357	+
	358	+ if (skb && frag_size) {
	359	+ skb->head_frag = 1;
	360	+ if (page_is_pfmemalloc(virt_to_head_page(data)))
	361	+ skb->pfmemalloc = 1;
	362	+ }
	363	+ return skb;
	364	+}
	365	+EXPORT_SYMBOL(build_skb_around);
	366	+
324	367	#define NAPI_SKB_CACHE_SIZE 64
325	368
326	369	struct napi_alloc_cache {
..	..	@@ -331,21 +374,21 @@
331	374
332	375	static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
333	376	static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
334		-static DEFINE_LOCAL_IRQ_LOCK(netdev_alloc_lock);
335		-static DEFINE_LOCAL_IRQ_LOCK(napi_alloc_cache_lock);
336	377
337		-static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
	378	+static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
338	379	{
339		- struct page_frag_cache *nc;
340		- unsigned long flags;
341		- void *data;
	380	+ struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
342	381
343		- local_lock_irqsave(netdev_alloc_lock, flags);
344		- nc = this_cpu_ptr(&netdev_alloc_cache);
345		- data = page_frag_alloc(nc, fragsz, gfp_mask);
346		- local_unlock_irqrestore(netdev_alloc_lock, flags);
347		- return data;
	382	+ return page_frag_alloc(&nc->page, fragsz, gfp_mask);
348	383	}
	384	+
	385	+void *napi_alloc_frag(unsigned int fragsz)
	386	+{
	387	+ fragsz = SKB_DATA_ALIGN(fragsz);
	388	+
	389	+ return __napi_alloc_frag(fragsz, GFP_ATOMIC);
	390	+}
	391	+EXPORT_SYMBOL(napi_alloc_frag);
349	392
350	393	/**
351	394	* netdev_alloc_frag - allocate a page fragment
..	..	@@ -356,30 +399,21 @@
356	399	*/
357	400	void *netdev_alloc_frag(unsigned int fragsz)
358	401	{
359		- fragsz = SKB_DATA_ALIGN(fragsz);
360		-
361		- return __netdev_alloc_frag(fragsz, GFP_ATOMIC);
362		-}
363		-EXPORT_SYMBOL(netdev_alloc_frag);
364		-
365		-static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
366		-{
367		- struct napi_alloc_cache *nc;
	402	+ struct page_frag_cache *nc;
368	403	void *data;
369	404
370		- nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
371		- data = page_frag_alloc(&nc->page, fragsz, gfp_mask);
372		- put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
	405	+ fragsz = SKB_DATA_ALIGN(fragsz);
	406	+ if (in_irq() \|\| irqs_disabled()) {
	407	+ nc = this_cpu_ptr(&netdev_alloc_cache);
	408	+ data = page_frag_alloc(nc, fragsz, GFP_ATOMIC);
	409	+ } else {
	410	+ local_bh_disable();
	411	+ data = __napi_alloc_frag(fragsz, GFP_ATOMIC);
	412	+ local_bh_enable();
	413	+ }
373	414	return data;
374	415	}
375		-
376		-void *napi_alloc_frag(unsigned int fragsz)
377		-{
378		- fragsz = SKB_DATA_ALIGN(fragsz);
379		-
380		- return __napi_alloc_frag(fragsz, GFP_ATOMIC);
381		-}
382		-EXPORT_SYMBOL(napi_alloc_frag);
	416	+EXPORT_SYMBOL(netdev_alloc_frag);
383	417
384	418	/**
385	419	* __netdev_alloc_skb - allocate an skbuff for rx on a specific device
..	..	@@ -398,7 +432,6 @@
398	432	gfp_t gfp_mask)
399	433	{
400	434	struct page_frag_cache *nc;
401		- unsigned long flags;
402	435	struct sk_buff *skb;
403	436	bool pfmemalloc;
404	437	void *data;
..	..	@@ -423,13 +456,17 @@
423	456	if (sk_memalloc_socks())
424	457	gfp_mask \|= __GFP_MEMALLOC;
425	458
426		- local_lock_irqsave(netdev_alloc_lock, flags);
427		-
428		- nc = this_cpu_ptr(&netdev_alloc_cache);
429		- data = page_frag_alloc(nc, len, gfp_mask);
430		- pfmemalloc = nc->pfmemalloc;
431		-
432		- local_unlock_irqrestore(netdev_alloc_lock, flags);
	459	+ if (in_irq() \|\| irqs_disabled()) {
	460	+ nc = this_cpu_ptr(&netdev_alloc_cache);
	461	+ data = page_frag_alloc(nc, len, gfp_mask);
	462	+ pfmemalloc = nc->pfmemalloc;
	463	+ } else {
	464	+ local_bh_disable();
	465	+ nc = this_cpu_ptr(&napi_alloc_cache.page);
	466	+ data = page_frag_alloc(nc, len, gfp_mask);
	467	+ pfmemalloc = nc->pfmemalloc;
	468	+ local_bh_enable();
	469	+ }
433	470
434	471	if (unlikely(!data))
435	472	return NULL;
..	..	@@ -440,7 +477,6 @@
440	477	return NULL;
441	478	}
442	479
443		- /* use OR instead of assignment to avoid clearing of bits in mask */
444	480	if (pfmemalloc)
445	481	skb->pfmemalloc = 1;
446	482	skb->head_frag = 1;
..	..	@@ -473,7 +509,6 @@
473	509	struct napi_alloc_cache *nc;
474	510	struct sk_buff *skb;
475	511	void *data;
476		- bool pfmemalloc;
477	512
478	513	len += NET_SKB_PAD + NET_IP_ALIGN;
479	514
..	..	@@ -496,10 +531,7 @@
496	531	if (sk_memalloc_socks())
497	532	gfp_mask \|= __GFP_MEMALLOC;
498	533
499		- nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
500	534	data = page_frag_alloc(&nc->page, len, gfp_mask);
501		- pfmemalloc = nc->page.pfmemalloc;
502		- put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
503	535	if (unlikely(!data))
504	536	return NULL;
505	537
..	..	@@ -509,8 +541,7 @@
509	541	return NULL;
510	542	}
511	543
512		- /* use OR instead of assignment to avoid clearing of bits in mask */
513		- if (pfmemalloc)
	544	+ if (nc->page.pfmemalloc)
514	545	skb->pfmemalloc = 1;
515	546	skb->head_frag = 1;
516	547
..	..	@@ -629,8 +660,8 @@
629	660
630	661	void skb_release_head_state(struct sk_buff *skb)
631	662	{
	663	+ nf_reset_ct(skb);
632	664	skb_dst_drop(skb);
633		- secpath_reset(skb);
634	665	if (skb->destructor) {
635	666	WARN_ON(in_irq());
636	667	skb->destructor(skb);
..	..	@@ -638,9 +669,7 @@
638	669	#if IS_ENABLED(CONFIG_NF_CONNTRACK)
639	670	nf_conntrack_put(skb_nfct(skb));
640	671	#endif
641		-#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
642		- nf_bridge_put(skb->nf_bridge);
643		-#endif
	672	+ skb_ext_put(skb);
644	673	}
645	674
646	675	/* Free everything but the sk_buff shell. */
..	..	@@ -679,6 +708,7 @@
679	708	if (!skb_unref(skb))
680	709	return;
681	710
	711	+ trace_android_vh_kfree_skb(skb);
682	712	trace_kfree_skb(skb, __builtin_return_address(0));
683	713	__kfree_skb(skb);
684	714	}
..	..	@@ -695,6 +725,101 @@
695	725	}
696	726	EXPORT_SYMBOL(kfree_skb_list);
697	727
	728	+/* Dump skb information and contents.
	729	+ *
	730	+ * Must only be called from net_ratelimit()-ed paths.
	731	+ *
	732	+ * Dumps whole packets if full_pkt, only headers otherwise.
	733	+ */
	734	+void skb_dump(const char level, const struct sk_buff skb, bool full_pkt)
	735	+{
	736	+ struct skb_shared_info *sh = skb_shinfo(skb);
	737	+ struct net_device *dev = skb->dev;
	738	+ struct sock *sk = skb->sk;
	739	+ struct sk_buff *list_skb;
	740	+ bool has_mac, has_trans;
	741	+ int headroom, tailroom;
	742	+ int i, len, seg_len;
	743	+
	744	+ if (full_pkt)
	745	+ len = skb->len;
	746	+ else
	747	+ len = min_t(int, skb->len, MAX_HEADER + 128);
	748	+
	749	+ headroom = skb_headroom(skb);
	750	+ tailroom = skb_tailroom(skb);
	751	+
	752	+ has_mac = skb_mac_header_was_set(skb);
	753	+ has_trans = skb_transport_header_was_set(skb);
	754	+
	755	+ printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n"
	756	+ "mac=(%d,%d) net=(%d,%d) trans=%d\n"
	757	+ "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n"
	758	+ "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n"
	759	+ "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n",
	760	+ level, skb->len, headroom, skb_headlen(skb), tailroom,
	761	+ has_mac ? skb->mac_header : -1,
	762	+ has_mac ? skb_mac_header_len(skb) : -1,
	763	+ skb->network_header,
	764	+ has_trans ? skb_network_header_len(skb) : -1,
	765	+ has_trans ? skb->transport_header : -1,
	766	+ sh->tx_flags, sh->nr_frags,
	767	+ sh->gso_size, sh->gso_type, sh->gso_segs,
	768	+ skb->csum, skb->ip_summed, skb->csum_complete_sw,
	769	+ skb->csum_valid, skb->csum_level,
	770	+ skb->hash, skb->sw_hash, skb->l4_hash,
	771	+ ntohs(skb->protocol), skb->pkt_type, skb->skb_iif);
	772	+
	773	+ if (dev)
	774	+ printk("%sdev name=%s feat=%pNF\n",
	775	+ level, dev->name, &dev->features);
	776	+ if (sk)
	777	+ printk("%ssk family=%hu type=%u proto=%u\n",
	778	+ level, sk->sk_family, sk->sk_type, sk->sk_protocol);
	779	+
	780	+ if (full_pkt && headroom)
	781	+ print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET,
	782	+ 16, 1, skb->head, headroom, false);
	783	+
	784	+ seg_len = min_t(int, skb_headlen(skb), len);
	785	+ if (seg_len)
	786	+ print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET,
	787	+ 16, 1, skb->data, seg_len, false);
	788	+ len -= seg_len;
	789	+
	790	+ if (full_pkt && tailroom)
	791	+ print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET,
	792	+ 16, 1, skb_tail_pointer(skb), tailroom, false);
	793	+
	794	+ for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) {
	795	+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
	796	+ u32 p_off, p_len, copied;
	797	+ struct page *p;
	798	+ u8 *vaddr;
	799	+
	800	+ skb_frag_foreach_page(frag, skb_frag_off(frag),
	801	+ skb_frag_size(frag), p, p_off, p_len,
	802	+ copied) {
	803	+ seg_len = min_t(int, p_len, len);
	804	+ vaddr = kmap_atomic(p);
	805	+ print_hex_dump(level, "skb frag: ",
	806	+ DUMP_PREFIX_OFFSET,
	807	+ 16, 1, vaddr + p_off, seg_len, false);
	808	+ kunmap_atomic(vaddr);
	809	+ len -= seg_len;
	810	+ if (!len)
	811	+ break;
	812	+ }
	813	+ }
	814	+
	815	+ if (full_pkt && skb_has_frag_list(skb)) {
	816	+ printk("skb fraglist:\n");
	817	+ skb_walk_frags(skb, list_skb)
	818	+ skb_dump(level, list_skb, true);
	819	+ }
	820	+}
	821	+EXPORT_SYMBOL(skb_dump);
	822	+
698	823	/**
699	824	* skb_tx_error - report an sk_buff xmit error
700	825	* @skb: buffer that triggered an error
..	..	@@ -708,6 +833,7 @@
708	833	}
709	834	EXPORT_SYMBOL(skb_tx_error);
710	835
	836	+#ifdef CONFIG_TRACEPOINTS
711	837	/**
712	838	* consume_skb - free an skbuff
713	839	* @skb: buffer to free
..	..	@@ -725,6 +851,7 @@
725	851	__kfree_skb(skb);
726	852	}
727	853	EXPORT_SYMBOL(consume_skb);
	854	+#endif
728	855
729	856	/**
730	857	* consume_stateless_skb - free an skbuff, assuming it is stateless
..	..	@@ -742,26 +869,23 @@
742	869
743	870	void __kfree_skb_flush(void)
744	871	{
745		- struct napi_alloc_cache *nc;
	872	+ struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
746	873
747		- nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
748	874	/* flush skb_cache if containing objects */
749	875	if (nc->skb_count) {
750	876	kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
751	877	nc->skb_cache);
752	878	nc->skb_count = 0;
753	879	}
754		- put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
755	880	}
756	881
757	882	static inline void _kfree_skb_defer(struct sk_buff *skb)
758	883	{
759		- struct napi_alloc_cache *nc;
	884	+ struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
760	885
761	886	/* drop skb->head and call any destructors for packet */
762	887	skb_release_all(skb);
763	888
764		- nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
765	889	/* record skb to CPU local list */
766	890	nc->skb_cache[nc->skb_count++] = skb;
767	891
..	..	@@ -776,7 +900,6 @@
776	900	nc->skb_cache);
777	901	nc->skb_count = 0;
778	902	}
779		- put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
780	903	}
781	904	void __kfree_skb_defer(struct sk_buff *skb)
782	905	{
..	..	@@ -785,9 +908,6 @@
785	908
786	909	void napi_consume_skb(struct sk_buff *skb, int budget)
787	910	{
788		- if (unlikely(!skb))
789		- return;
790		-
791	911	/* Zero budget indicate non-NAPI context called us, like netpoll */
792	912	if (unlikely(!budget)) {
793	913	dev_consume_skb_any(skb);
..	..	@@ -824,9 +944,7 @@
824	944	new->dev = old->dev;
825	945	memcpy(new->cb, old->cb, sizeof(old->cb));
826	946	skb_dst_copy(new, old);
827		-#ifdef CONFIG_XFRM
828		- new->sp = secpath_get(old->sp);
829		-#endif
	947	+ __skb_ext_copy(new, old);
830	948	__nf_copy(new, old, false);
831	949
832	950	/* Note : this field could be in headers_start/headers_end section
..	..	@@ -902,6 +1020,31 @@
902	1020	return n;
903	1021	#undef C
904	1022	}
	1023	+
	1024	+/**
	1025	+ * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg
	1026	+ * @first: first sk_buff of the msg
	1027	+ */
	1028	+struct sk_buff alloc_skb_for_msg(struct sk_buff first)
	1029	+{
	1030	+ struct sk_buff *n;
	1031	+
	1032	+ n = alloc_skb(0, GFP_ATOMIC);
	1033	+ if (!n)
	1034	+ return NULL;
	1035	+
	1036	+ n->len = first->len;
	1037	+ n->data_len = first->len;
	1038	+ n->truesize = first->truesize;
	1039	+
	1040	+ skb_shinfo(n)->frag_list = first;
	1041	+
	1042	+ __copy_skb_header(n, first);
	1043	+ n->destructor = NULL;
	1044	+
	1045	+ return n;
	1046	+}
	1047	+EXPORT_SYMBOL_GPL(alloc_skb_for_msg);
905	1048
906	1049	/**
907	1050	* skb_morph - morph one skb into another
..	..	@@ -1027,7 +1170,11 @@
1027	1170	uarg->len++;
1028	1171	uarg->bytelen = bytelen;
1029	1172	atomic_set(&sk->sk_zckey, ++next);
1030		- sock_zerocopy_get(uarg);
	1173	+
	1174	+ /* no extra ref when appending to datagram (MSG_MORE) */
	1175	+ if (sk->sk_type == SOCK_STREAM)
	1176	+ sock_zerocopy_get(uarg);
	1177	+
1031	1178	return uarg;
1032	1179	}
1033	1180	}
..	..	@@ -1117,7 +1264,7 @@
1117	1264	}
1118	1265	EXPORT_SYMBOL_GPL(sock_zerocopy_put);
1119	1266
1120		-void sock_zerocopy_put_abort(struct ubuf_info *uarg)
	1267	+void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
1121	1268	{
1122	1269	if (uarg) {
1123	1270	struct sock *sk = skb_from_uarg(uarg)->sk;
..	..	@@ -1125,13 +1272,17 @@
1125	1272	atomic_dec(&sk->sk_zckey);
1126	1273	uarg->len--;
1127	1274
1128		- sock_zerocopy_put(uarg);
	1275	+ if (have_uref)
	1276	+ sock_zerocopy_put(uarg);
1129	1277	}
1130	1278	}
1131	1279	EXPORT_SYMBOL_GPL(sock_zerocopy_put_abort);
1132	1280
1133		-extern int __zerocopy_sg_from_iter(struct sock sk, struct sk_buff skb,
1134		- struct iov_iter *from, size_t length);
	1281	+int skb_zerocopy_iter_dgram(struct sk_buff skb, struct msghdr msg, int len)
	1282	+{
	1283	+ return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);
	1284	+}
	1285	+EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);
1135	1286
1136	1287	int skb_zerocopy_iter_stream(struct sock sk, struct sk_buff skb,
1137	1288	struct msghdr *msg, int len,
..	..	@@ -1159,7 +1310,7 @@
1159	1310	return err;
1160	1311	}
1161	1312
1162		- skb_zcopy_set(skb, uarg);
	1313	+ skb_zcopy_set(skb, uarg, NULL);
1163	1314	return skb->len - orig_len;
1164	1315	}
1165	1316	EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);
..	..	@@ -1179,7 +1330,7 @@
1179	1330	if (skb_copy_ubufs(nskb, GFP_ATOMIC))
1180	1331	return -EIO;
1181	1332	}
1182		- skb_zcopy_set(nskb, skb_uarg(orig));
	1333	+ skb_zcopy_set(nskb, skb_uarg(orig), NULL);
1183	1334	}
1184	1335	return 0;
1185	1336	}
..	..	@@ -1235,7 +1386,7 @@
1235	1386	struct page *p;
1236	1387	u8 *vaddr;
1237	1388
1238		- skb_frag_foreach_page(f, f->page_offset, skb_frag_size(f),
	1389	+ skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
1239	1390	p, p_off, p_len, copied) {
1240	1391	u32 copy, done = 0;
1241	1392	vaddr = kmap_atomic(p);
..	..	@@ -1525,11 +1676,10 @@
1525	1676	skb->head = data;
1526	1677	skb->head_frag = 0;
1527	1678	skb->data += off;
	1679	+
	1680	+ skb_set_end_offset(skb, size);
1528	1681	#ifdef NET_SKBUFF_DATA_USES_OFFSET
1529		- skb->end = size;
1530	1682	off = nhead;
1531		-#else
1532		- skb->end = skb->head + size;
1533	1683	#endif
1534	1684	skb->tail += off;
1535	1685	skb_headers_offset_update(skb, nhead);
..	..	@@ -1576,6 +1726,38 @@
1576	1726	return skb2;
1577	1727	}
1578	1728	EXPORT_SYMBOL(skb_realloc_headroom);
	1729	+
	1730	+int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri)
	1731	+{
	1732	+ unsigned int saved_end_offset, saved_truesize;
	1733	+ struct skb_shared_info *shinfo;
	1734	+ int res;
	1735	+
	1736	+ saved_end_offset = skb_end_offset(skb);
	1737	+ saved_truesize = skb->truesize;
	1738	+
	1739	+ res = pskb_expand_head(skb, 0, 0, pri);
	1740	+ if (res)
	1741	+ return res;
	1742	+
	1743	+ skb->truesize = saved_truesize;
	1744	+
	1745	+ if (likely(skb_end_offset(skb) == saved_end_offset))
	1746	+ return 0;
	1747	+
	1748	+ shinfo = skb_shinfo(skb);
	1749	+
	1750	+ /* We are about to change back skb->end,
	1751	+ * we need to move skb_shinfo() to its new location.
	1752	+ */
	1753	+ memmove(skb->head + saved_end_offset,
	1754	+ shinfo,
	1755	+ offsetof(struct skb_shared_info, frags[shinfo->nr_frags]));
	1756	+
	1757	+ skb_set_end_offset(skb, saved_end_offset);
	1758	+
	1759	+ return 0;
	1760	+}
1579	1761
1580	1762	/**
1581	1763	* skb_copy_expand - copy and expand sk_buff
..	..	@@ -1959,8 +2141,6 @@
1959	2141	struct sk_buff *insp = NULL;
1960	2142
1961	2143	do {
1962		- BUG_ON(!list);
1963		-
1964	2144	if (list->len <= eat) {
1965	2145	/* Eaten as whole. */
1966	2146	eat -= list->len;
..	..	@@ -2012,10 +2192,12 @@
2012	2192	skb_frag_unref(skb, i);
2013	2193	eat -= size;
2014	2194	} else {
2015		- skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
	2195	+ skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
	2196	+
	2197	+ *frag = skb_shinfo(skb)->frags[i];
2016	2198	if (eat) {
2017		- skb_shinfo(skb)->frags[k].page_offset += eat;
2018		- skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat);
	2199	+ skb_frag_off_add(frag, eat);
	2200	+ skb_frag_size_sub(frag, eat);
2019	2201	if (!i)
2020	2202	goto end;
2021	2203	eat = 0;
..	..	@@ -2087,7 +2269,7 @@
2087	2269	copy = len;
2088	2270
2089	2271	skb_frag_foreach_page(f,
2090		- f->page_offset + offset - start,
	2272	+ skb_frag_off(f) + offset - start,
2091	2273	copy, p, p_off, p_len, copied) {
2092	2274	vaddr = kmap_atomic(p);
2093	2275	memcpy(to + copied, vaddr + p_off, p_len);
..	..	@@ -2263,7 +2445,7 @@
2263	2445	const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
2264	2446
2265	2447	if (__splice_segment(skb_frag_page(f),
2266		- f->page_offset, skb_frag_size(f),
	2448	+ skb_frag_off(f), skb_frag_size(f),
2267	2449	offset, len, spd, false, sk, pipe))
2268	2450	return true;
2269	2451	}
..	..	@@ -2353,20 +2535,20 @@
2353	2535	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2354	2536	skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2355	2537
2356		- if (offset < frag->size)
	2538	+ if (offset < skb_frag_size(frag))
2357	2539	break;
2358	2540
2359		- offset -= frag->size;
	2541	+ offset -= skb_frag_size(frag);
2360	2542	}
2361	2543
2362	2544	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
2363	2545	skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx];
2364	2546
2365		- slen = min_t(size_t, len, frag->size - offset);
	2547	+ slen = min_t(size_t, len, skb_frag_size(frag) - offset);
2366	2548
2367	2549	while (slen) {
2368		- ret = kernel_sendpage_locked(sk, frag->page.p,
2369		- frag->page_offset + offset,
	2550	+ ret = kernel_sendpage_locked(sk, skb_frag_page(frag),
	2551	+ skb_frag_off(frag) + offset,
2370	2552	slen, MSG_DONTWAIT);
2371	2553	if (ret <= 0)
2372	2554	goto error;
..	..	@@ -2400,19 +2582,6 @@
2400	2582	return orig_len == len ? ret : orig_len - len;
2401	2583	}
2402	2584	EXPORT_SYMBOL_GPL(skb_send_sock_locked);
2403		-
2404		-/* Send skb data on a socket. */
2405		-int skb_send_sock(struct sock sk, struct sk_buff skb, int offset, int len)
2406		-{
2407		- int ret = 0;
2408		-
2409		- lock_sock(sk);
2410		- ret = skb_send_sock_locked(sk, skb, offset, len);
2411		- release_sock(sk);
2412		-
2413		- return ret;
2414		-}
2415		-EXPORT_SYMBOL_GPL(skb_send_sock);
2416	2585
2417	2586	/**
2418	2587	* skb_store_bits - store bits from kernel buffer to skb
..	..	@@ -2461,7 +2630,7 @@
2461	2630	copy = len;
2462	2631
2463	2632	skb_frag_foreach_page(frag,
2464		- frag->page_offset + offset - start,
	2633	+ skb_frag_off(frag) + offset - start,
2465	2634	copy, p, p_off, p_len, copied) {
2466	2635	vaddr = kmap_atomic(p);
2467	2636	memcpy(vaddr + p_off, from + copied, p_len);
..	..	@@ -2516,7 +2685,8 @@
2516	2685	if (copy > 0) {
2517	2686	if (copy > len)
2518	2687	copy = len;
2519		- csum = ops->update(skb->data + offset, copy, csum);
	2688	+ csum = INDIRECT_CALL_1(ops->update, csum_partial_ext,
	2689	+ skb->data + offset, copy, csum);
2520	2690	if ((len -= copy) == 0)
2521	2691	return csum;
2522	2692	offset += copy;
..	..	@@ -2540,12 +2710,16 @@
2540	2710	copy = len;
2541	2711
2542	2712	skb_frag_foreach_page(frag,
2543		- frag->page_offset + offset - start,
	2713	+ skb_frag_off(frag) + offset - start,
2544	2714	copy, p, p_off, p_len, copied) {
2545	2715	vaddr = kmap_atomic(p);
2546		- csum2 = ops->update(vaddr + p_off, p_len, 0);
	2716	+ csum2 = INDIRECT_CALL_1(ops->update,
	2717	+ csum_partial_ext,
	2718	+ vaddr + p_off, p_len, 0);
2547	2719	kunmap_atomic(vaddr);
2548		- csum = ops->combine(csum, csum2, pos, p_len);
	2720	+ csum = INDIRECT_CALL_1(ops->combine,
	2721	+ csum_block_add_ext, csum,
	2722	+ csum2, pos, p_len);
2549	2723	pos += p_len;
2550	2724	}
2551	2725
..	..	@@ -2568,7 +2742,8 @@
2568	2742	copy = len;
2569	2743	csum2 = __skb_checksum(frag_iter, offset - start,
2570	2744	copy, 0, ops);
2571		- csum = ops->combine(csum, csum2, pos, copy);
	2745	+ csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext,
	2746	+ csum, csum2, pos, copy);
2572	2747	if ((len -= copy) == 0)
2573	2748	return csum;
2574	2749	offset += copy;
..	..	@@ -2597,19 +2772,20 @@
2597	2772	/* Both of above in one bottle. */
2598	2773
2599	2774	__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
2600		- u8 *to, int len, __wsum csum)
	2775	+ u8 *to, int len)
2601	2776	{
2602	2777	int start = skb_headlen(skb);
2603	2778	int i, copy = start - offset;
2604	2779	struct sk_buff *frag_iter;
2605	2780	int pos = 0;
	2781	+ __wsum csum = 0;
2606	2782
2607	2783	/* Copy header. */
2608	2784	if (copy > 0) {
2609	2785	if (copy > len)
2610	2786	copy = len;
2611	2787	csum = csum_partial_copy_nocheck(skb->data + offset, to,
2612		- copy, csum);
	2788	+ copy);
2613	2789	if ((len -= copy) == 0)
2614	2790	return csum;
2615	2791	offset += copy;
..	..	@@ -2634,12 +2810,12 @@
2634	2810	copy = len;
2635	2811
2636	2812	skb_frag_foreach_page(frag,
2637		- frag->page_offset + offset - start,
	2813	+ skb_frag_off(frag) + offset - start,
2638	2814	copy, p, p_off, p_len, copied) {
2639	2815	vaddr = kmap_atomic(p);
2640	2816	csum2 = csum_partial_copy_nocheck(vaddr + p_off,
2641	2817	to + copied,
2642		- p_len, 0);
	2818	+ p_len);
2643	2819	kunmap_atomic(vaddr);
2644	2820	csum = csum_block_add(csum, csum2, pos);
2645	2821	pos += p_len;
..	..	@@ -2665,7 +2841,7 @@
2665	2841	copy = len;
2666	2842	csum2 = skb_copy_and_csum_bits(frag_iter,
2667	2843	offset - start,
2668		- to, copy, 0);
	2844	+ to, copy);
2669	2845	csum = csum_block_add(csum, csum2, pos);
2670	2846	if ((len -= copy) == 0)
2671	2847	return csum;
..	..	@@ -2679,6 +2855,65 @@
2679	2855	return csum;
2680	2856	}
2681	2857	EXPORT_SYMBOL(skb_copy_and_csum_bits);
	2858	+
	2859	+__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)
	2860	+{
	2861	+ __sum16 sum;
	2862	+
	2863	+ sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));
	2864	+ /* See comments in __skb_checksum_complete(). */
	2865	+ if (likely(!sum)) {
	2866	+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
	2867	+ !skb->csum_complete_sw)
	2868	+ netdev_rx_csum_fault(skb->dev, skb);
	2869	+ }
	2870	+ if (!skb_shared(skb))
	2871	+ skb->csum_valid = !sum;
	2872	+ return sum;
	2873	+}
	2874	+EXPORT_SYMBOL(__skb_checksum_complete_head);
	2875	+
	2876	+/* This function assumes skb->csum already holds pseudo header's checksum,
	2877	+ * which has been changed from the hardware checksum, for example, by
	2878	+ * __skb_checksum_validate_complete(). And, the original skb->csum must
	2879	+ * have been validated unsuccessfully for CHECKSUM_COMPLETE case.
	2880	+ *
	2881	+ * It returns non-zero if the recomputed checksum is still invalid, otherwise
	2882	+ * zero. The new checksum is stored back into skb->csum unless the skb is
	2883	+ * shared.
	2884	+ */
	2885	+__sum16 __skb_checksum_complete(struct sk_buff *skb)
	2886	+{
	2887	+ __wsum csum;
	2888	+ __sum16 sum;
	2889	+
	2890	+ csum = skb_checksum(skb, 0, skb->len, 0);
	2891	+
	2892	+ sum = csum_fold(csum_add(skb->csum, csum));
	2893	+ /* This check is inverted, because we already knew the hardware
	2894	+ * checksum is invalid before calling this function. So, if the
	2895	+ * re-computed checksum is valid instead, then we have a mismatch
	2896	+ * between the original skb->csum and skb_checksum(). This means either
	2897	+ * the original hardware checksum is incorrect or we screw up skb->csum
	2898	+ * when moving skb->data around.
	2899	+ */
	2900	+ if (likely(!sum)) {
	2901	+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
	2902	+ !skb->csum_complete_sw)
	2903	+ netdev_rx_csum_fault(skb->dev, skb);
	2904	+ }
	2905	+
	2906	+ if (!skb_shared(skb)) {
	2907	+ /* Save full packet checksum */
	2908	+ skb->csum = csum;
	2909	+ skb->ip_summed = CHECKSUM_COMPLETE;
	2910	+ skb->csum_complete_sw = 1;
	2911	+ skb->csum_valid = !sum;
	2912	+ }
	2913	+
	2914	+ return sum;
	2915	+}
	2916	+EXPORT_SYMBOL(__skb_checksum_complete);
2682	2917
2683	2918	static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum)
2684	2919	{
..	..	@@ -2794,11 +3029,15 @@
2794	3029	skb_zerocopy_clone(to, from, GFP_ATOMIC);
2795	3030
2796	3031	for (i = 0; i < skb_shinfo(from)->nr_frags; i++) {
	3032	+ int size;
	3033	+
2797	3034	if (!len)
2798	3035	break;
2799	3036	skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i];
2800		- skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len);
2801		- len -= skb_shinfo(to)->frags[j].size;
	3037	+ size = min_t(int, skb_frag_size(&skb_shinfo(to)->frags[j]),
	3038	+ len);
	3039	+ skb_frag_size_set(&skb_shinfo(to)->frags[j], size);
	3040	+ len -= size;
2802	3041	skb_frag_ref(to, j);
2803	3042	j++;
2804	3043	}
..	..	@@ -2825,7 +3064,7 @@
2825	3064	csum = 0;
2826	3065	if (csstart != skb->len)
2827	3066	csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
2828		- skb->len - csstart, 0);
	3067	+ skb->len - csstart);
2829	3068
2830	3069	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2831	3070	long csstuff = csstart + skb->csum_offset;
..	..	@@ -3000,28 +3239,6 @@
3000	3239	}
3001	3240	EXPORT_SYMBOL(skb_append);
3002	3241
3003		-/**
3004		- * skb_insert - insert a buffer
3005		- * @old: buffer to insert before
3006		- * @newsk: buffer to insert
3007		- * @list: list to use
3008		- *
3009		- * Place a packet before a given packet in a list. The list locks are
3010		- * taken and this function is atomic with respect to other list locked
3011		- * calls.
3012		- *
3013		- * A buffer cannot be placed on two lists at the same time.
3014		- */
3015		-void skb_insert(struct sk_buff old, struct sk_buff newsk, struct sk_buff_head *list)
3016		-{
3017		- unsigned long flags;
3018		-
3019		- spin_lock_irqsave(&list->lock, flags);
3020		- __skb_insert(newsk, old->prev, old, list);
3021		- spin_unlock_irqrestore(&list->lock, flags);
3022		-}
3023		-EXPORT_SYMBOL(skb_insert);
3024		-
3025	3242	static inline void skb_split_inside_header(struct sk_buff *skb,
3026	3243	struct sk_buff* skb1,
3027	3244	const u32 len, const int pos)
..	..	@@ -3071,7 +3288,7 @@
3071	3288	* 2. Split is accurately. We make this.
3072	3289	*/
3073	3290	skb_frag_ref(skb, i);
3074		- skb_shinfo(skb1)->frags[0].page_offset += len - pos;
	3291	+ skb_frag_off_add(&skb_shinfo(skb1)->frags[0], len - pos);
3075	3292	skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos);
3076	3293	skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos);
3077	3294	skb_shinfo(skb)->nr_frags++;
..	..	@@ -3110,19 +3327,7 @@
3110	3327	*/
3111	3328	static int skb_prepare_for_shift(struct sk_buff *skb)
3112	3329	{
3113		- int ret = 0;
3114		-
3115		- if (skb_cloned(skb)) {
3116		- /* Save and restore truesize: pskb_expand_head() may reallocate
3117		- * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we
3118		- * cannot change truesize at this point.
3119		- */
3120		- unsigned int save_truesize = skb->truesize;
3121		-
3122		- ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3123		- skb->truesize = save_truesize;
3124		- }
3125		- return ret;
	3330	+ return skb_unclone_keeptruesize(skb, GFP_ATOMIC);
3126	3331	}
3127	3332
3128	3333	/**
..	..	@@ -3146,7 +3351,7 @@
3146	3351	int skb_shift(struct sk_buff tgt, struct sk_buff skb, int shiftlen)
3147	3352	{
3148	3353	int from, to, merge, todo;
3149		- struct skb_frag_struct fragfrom, fragto;
	3354	+ skb_frag_t fragfrom, fragto;
3150	3355
3151	3356	BUG_ON(shiftlen > skb->len);
3152	3357
..	..	@@ -3165,7 +3370,7 @@
3165	3370	*/
3166	3371	if (!to \|\|
3167	3372	!skb_can_coalesce(tgt, to, skb_frag_page(fragfrom),
3168		- fragfrom->page_offset)) {
	3373	+ skb_frag_off(fragfrom))) {
3169	3374	merge = -1;
3170	3375	} else {
3171	3376	merge = to - 1;
..	..	@@ -3182,7 +3387,7 @@
3182	3387
3183	3388	skb_frag_size_add(fragto, shiftlen);
3184	3389	skb_frag_size_sub(fragfrom, shiftlen);
3185		- fragfrom->page_offset += shiftlen;
	3390	+ skb_frag_off_add(fragfrom, shiftlen);
3186	3391
3187	3392	goto onlymerged;
3188	3393	}
..	..	@@ -3213,11 +3418,11 @@
3213	3418
3214	3419	} else {
3215	3420	__skb_frag_ref(fragfrom);
3216		- fragto->page = fragfrom->page;
3217		- fragto->page_offset = fragfrom->page_offset;
	3421	+ skb_frag_page_copy(fragto, fragfrom);
	3422	+ skb_frag_off_copy(fragto, fragfrom);
3218	3423	skb_frag_size_set(fragto, todo);
3219	3424
3220		- fragfrom->page_offset += todo;
	3425	+ skb_frag_off_add(fragfrom, todo);
3221	3426	skb_frag_size_sub(fragfrom, todo);
3222	3427	todo = 0;
3223	3428
..	..	@@ -3342,7 +3547,7 @@
3342	3547	if (!st->frag_data)
3343	3548	st->frag_data = kmap_atomic(skb_frag_page(frag));
3344	3549
3345		- data = (u8 ) st->frag_data + frag->page_offset +
	3550	+ data = (u8 ) st->frag_data + skb_frag_off(frag) +
3346	3551	(abs_offset - st->stepped_offset);
3347	3552
3348	3553	return block_limit - abs_offset;
..	..	@@ -3432,64 +3637,6 @@
3432	3637	}
3433	3638	EXPORT_SYMBOL(skb_find_text);
3434	3639
3435		-/**
3436		- * skb_append_datato_frags - append the user data to a skb
3437		- * @sk: sock structure
3438		- * @skb: skb structure to be appended with user data.
3439		- * @getfrag: call back function to be used for getting the user data
3440		- * @from: pointer to user message iov
3441		- * @length: length of the iov message
3442		- *
3443		- * Description: This procedure append the user data in the fragment part
3444		- * of the skb if any page alloc fails user this procedure returns -ENOMEM
3445		- */
3446		-int skb_append_datato_frags(struct sock sk, struct sk_buff skb,
3447		- int (getfrag)(void from, char *to, int offset,
3448		- int len, int odd, struct sk_buff *skb),
3449		- void *from, int length)
3450		-{
3451		- int frg_cnt = skb_shinfo(skb)->nr_frags;
3452		- int copy;
3453		- int offset = 0;
3454		- int ret;
3455		- struct page_frag *pfrag = &current->task_frag;
3456		-
3457		- do {
3458		- /* Return error if we don't have space for new frag */
3459		- if (frg_cnt >= MAX_SKB_FRAGS)
3460		- return -EMSGSIZE;
3461		-
3462		- if (!sk_page_frag_refill(sk, pfrag))
3463		- return -ENOMEM;
3464		-
3465		- /* copy the user data to page */
3466		- copy = min_t(int, length, pfrag->size - pfrag->offset);
3467		-
3468		- ret = getfrag(from, page_address(pfrag->page) + pfrag->offset,
3469		- offset, copy, 0, skb);
3470		- if (ret < 0)
3471		- return -EFAULT;
3472		-
3473		- /* copy was successful so update the size parameters */
3474		- skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset,
3475		- copy);
3476		- frg_cnt++;
3477		- pfrag->offset += copy;
3478		- get_page(pfrag->page);
3479		-
3480		- skb->truesize += copy;
3481		- refcount_add(copy, &sk->sk_wmem_alloc);
3482		- skb->len += copy;
3483		- skb->data_len += copy;
3484		- offset += copy;
3485		- length -= copy;
3486		-
3487		- } while (length > 0);
3488		-
3489		- return 0;
3490		-}
3491		-EXPORT_SYMBOL(skb_append_datato_frags);
3492		-
3493	3640	int skb_append_pagefrags(struct sk_buff skb, struct page page,
3494	3641	int offset, size_t size)
3495	3642	{
..	..	@@ -3536,11 +3683,122 @@
3536	3683	struct page *page;
3537	3684
3538	3685	page = virt_to_head_page(frag_skb->head);
3539		- head_frag.page.p = page;
3540		- head_frag.page_offset = frag_skb->data -
3541		- (unsigned char *)page_address(page);
3542		- head_frag.size = skb_headlen(frag_skb);
	3686	+ __skb_frag_set_page(&head_frag, page);
	3687	+ skb_frag_off_set(&head_frag, frag_skb->data -
	3688	+ (unsigned char *)page_address(page));
	3689	+ skb_frag_size_set(&head_frag, skb_headlen(frag_skb));
3543	3690	return head_frag;
	3691	+}
	3692	+
	3693	+struct sk_buff skb_segment_list(struct sk_buff skb,
	3694	+ netdev_features_t features,
	3695	+ unsigned int offset)
	3696	+{
	3697	+ struct sk_buff *list_skb = skb_shinfo(skb)->frag_list;
	3698	+ unsigned int tnl_hlen = skb_tnl_header_len(skb);
	3699	+ unsigned int delta_truesize = 0;
	3700	+ unsigned int delta_len = 0;
	3701	+ struct sk_buff *tail = NULL;
	3702	+ struct sk_buff nskb, tmp;
	3703	+ int len_diff, err;
	3704	+
	3705	+ skb_push(skb, -skb_network_offset(skb) + offset);
	3706	+
	3707	+ skb_shinfo(skb)->frag_list = NULL;
	3708	+
	3709	+ do {
	3710	+ nskb = list_skb;
	3711	+ list_skb = list_skb->next;
	3712	+
	3713	+ err = 0;
	3714	+ delta_truesize += nskb->truesize;
	3715	+ if (skb_shared(nskb)) {
	3716	+ tmp = skb_clone(nskb, GFP_ATOMIC);
	3717	+ if (tmp) {
	3718	+ consume_skb(nskb);
	3719	+ nskb = tmp;
	3720	+ err = skb_unclone(nskb, GFP_ATOMIC);
	3721	+ } else {
	3722	+ err = -ENOMEM;
	3723	+ }
	3724	+ }
	3725	+
	3726	+ if (!tail)
	3727	+ skb->next = nskb;
	3728	+ else
	3729	+ tail->next = nskb;
	3730	+
	3731	+ if (unlikely(err)) {
	3732	+ nskb->next = list_skb;
	3733	+ goto err_linearize;
	3734	+ }
	3735	+
	3736	+ tail = nskb;
	3737	+
	3738	+ delta_len += nskb->len;
	3739	+
	3740	+ skb_push(nskb, -skb_network_offset(nskb) + offset);
	3741	+
	3742	+ skb_release_head_state(nskb);
	3743	+ len_diff = skb_network_header_len(nskb) - skb_network_header_len(skb);
	3744	+ __copy_skb_header(nskb, skb);
	3745	+
	3746	+ skb_headers_offset_update(nskb, skb_headroom(nskb) - skb_headroom(skb));
	3747	+ nskb->transport_header += len_diff;
	3748	+ skb_copy_from_linear_data_offset(skb, -tnl_hlen,
	3749	+ nskb->data - tnl_hlen,
	3750	+ offset + tnl_hlen);
	3751	+
	3752	+ if (skb_needs_linearize(nskb, features) &&
	3753	+ __skb_linearize(nskb))
	3754	+ goto err_linearize;
	3755	+
	3756	+ } while (list_skb);
	3757	+
	3758	+ skb->truesize = skb->truesize - delta_truesize;
	3759	+ skb->data_len = skb->data_len - delta_len;
	3760	+ skb->len = skb->len - delta_len;
	3761	+
	3762	+ skb_gso_reset(skb);
	3763	+
	3764	+ skb->prev = tail;
	3765	+
	3766	+ if (skb_needs_linearize(skb, features) &&
	3767	+ __skb_linearize(skb))
	3768	+ goto err_linearize;
	3769	+
	3770	+ skb_get(skb);
	3771	+
	3772	+ return skb;
	3773	+
	3774	+err_linearize:
	3775	+ kfree_skb_list(skb->next);
	3776	+ skb->next = NULL;
	3777	+ return ERR_PTR(-ENOMEM);
	3778	+}
	3779	+EXPORT_SYMBOL_GPL(skb_segment_list);
	3780	+
	3781	+int skb_gro_receive_list(struct sk_buff p, struct sk_buff skb)
	3782	+{
	3783	+ if (unlikely(p->len + skb->len >= 65536))
	3784	+ return -E2BIG;
	3785	+
	3786	+ if (NAPI_GRO_CB(p)->last == p)
	3787	+ skb_shinfo(p)->frag_list = skb;
	3788	+ else
	3789	+ NAPI_GRO_CB(p)->last->next = skb;
	3790	+
	3791	+ skb_pull(skb, skb_gro_offset(skb));
	3792	+
	3793	+ NAPI_GRO_CB(p)->last = skb;
	3794	+ NAPI_GRO_CB(p)->count++;
	3795	+ p->data_len += skb->len;
	3796	+ p->truesize += skb->truesize;
	3797	+ p->len += skb->len;
	3798	+
	3799	+ NAPI_GRO_CB(skb)->same_flow = 1;
	3800	+
	3801	+ return 0;
3544	3802	}
3545	3803
3546	3804	/**
..	..	@@ -3573,29 +3831,30 @@
3573	3831	int err = -ENOMEM;
3574	3832	int i = 0;
3575	3833	int pos;
3576		- int dummy;
3577	3834
3578		- if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) &&
3579		- (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) {
3580		- /* gso_size is untrusted, and we have a frag_list with a linear
3581		- * non head_frag head.
3582		- *
3583		- * (we assume checking the first list_skb member suffices;
3584		- * i.e if either of the list_skb members have non head_frag
3585		- * head, then the first one has too).
3586		- *
3587		- * If head_skb's headlen does not fit requested gso_size, it
3588		- * means that the frag_list members do NOT terminate on exact
3589		- * gso_size boundaries. Hence we cannot perform skb_frag_t page
3590		- * sharing. Therefore we must fallback to copying the frag_list
3591		- * skbs; we do so by disabling SG.
3592		- */
3593		- if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb))
3594		- features &= ~NETIF_F_SG;
	3835	+ if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) &&
	3836	+ mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) {
	3837	+ struct sk_buff *check_skb;
	3838	+
	3839	+ for (check_skb = list_skb; check_skb; check_skb = check_skb->next) {
	3840	+ if (skb_headlen(check_skb) && !check_skb->head_frag) {
	3841	+ /* gso_size is untrusted, and we have a frag_list with
	3842	+ * a linear non head_frag item.
	3843	+ *
	3844	+ * If head_skb's headlen does not fit requested gso_size,
	3845	+ * it means that the frag_list members do NOT terminate
	3846	+ * on exact gso_size boundaries. Hence we cannot perform
	3847	+ * skb_frag_t page sharing. Therefore we must fallback to
	3848	+ * copying the frag_list skbs; we do so by disabling SG.
	3849	+ */
	3850	+ features &= ~NETIF_F_SG;
	3851	+ break;
	3852	+ }
	3853	+ }
3595	3854	}
3596	3855
3597	3856	__skb_push(head_skb, doffset);
3598		- proto = skb_network_protocol(head_skb, &dummy);
	3857	+ proto = skb_network_protocol(head_skb, NULL);
3599	3858	if (unlikely(!proto))
3600	3859	return ERR_PTR(-EINVAL);
3601	3860
..	..	@@ -3741,14 +4000,20 @@
3741	4000	goto perform_csum_check;
3742	4001
3743	4002	if (!sg) {
3744		- if (!nskb->remcsum_offload)
3745		- nskb->ip_summed = CHECKSUM_NONE;
3746		- SKB_GSO_CB(nskb)->csum =
3747		- skb_copy_and_csum_bits(head_skb, offset,
3748		- skb_put(nskb, len),
3749		- len, 0);
3750		- SKB_GSO_CB(nskb)->csum_start =
3751		- skb_headroom(nskb) + doffset;
	4003	+ if (!csum) {
	4004	+ if (!nskb->remcsum_offload)
	4005	+ nskb->ip_summed = CHECKSUM_NONE;
	4006	+ SKB_GSO_CB(nskb)->csum =
	4007	+ skb_copy_and_csum_bits(head_skb, offset,
	4008	+ skb_put(nskb,
	4009	+ len),
	4010	+ len);
	4011	+ SKB_GSO_CB(nskb)->csum_start =
	4012	+ skb_headroom(nskb) + doffset;
	4013	+ } else {
	4014	+ if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len))
	4015	+ goto err;
	4016	+ }
3752	4017	continue;
3753	4018	}
3754	4019
..	..	@@ -3801,7 +4066,7 @@
3801	4066	size = skb_frag_size(nskb_frag);
3802	4067
3803	4068	if (pos < offset) {
3804		- nskb_frag->page_offset += offset - pos;
	4069	+ skb_frag_off_add(nskb_frag, offset - pos);
3805	4070	skb_frag_size_sub(nskb_frag, offset - pos);
3806	4071	}
3807	4072
..	..	@@ -3922,7 +4187,7 @@
3922	4187	--frag = --frag2;
3923	4188	} while (--i);
3924	4189
3925		- frag->page_offset += offset;
	4190	+ skb_frag_off_add(frag, offset);
3926	4191	skb_frag_size_sub(frag, offset);
3927	4192
3928	4193	/* all fragments truesize : remove (head size + sk_buff) */
..	..	@@ -3951,8 +4216,8 @@
3951	4216
3952	4217	pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
3953	4218
3954		- frag->page.p = page;
3955		- frag->page_offset = first_offset;
	4219	+ __skb_frag_set_page(frag, page);
	4220	+ skb_frag_off_set(frag, first_offset);
3956	4221	skb_frag_size_set(frag, first_size);
3957	4222
3958	4223	memcpy(frag + 1, skbinfo->frags, sizeof(frag) skbinfo->nr_frags);
..	..	@@ -3968,7 +4233,7 @@
3968	4233	if (offset > headlen) {
3969	4234	unsigned int eat = offset - headlen;
3970	4235
3971		- skbinfo->frags[0].page_offset += eat;
	4236	+ skb_frag_off_add(&skbinfo->frags[0], eat);
3972	4237	skb_frag_size_sub(&skbinfo->frags[0], eat);
3973	4238	skb->data_len -= eat;
3974	4239	skb->len -= eat;
..	..	@@ -3998,7 +4263,64 @@
3998	4263	NAPI_GRO_CB(skb)->same_flow = 1;
3999	4264	return 0;
4000	4265	}
4001		-EXPORT_SYMBOL_GPL(skb_gro_receive);
	4266	+
	4267	+#ifdef CONFIG_SKB_EXTENSIONS
	4268	+#define SKB_EXT_ALIGN_VALUE 8
	4269	+#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)
	4270	+
	4271	+static const u8 skb_ext_type_len[] = {
	4272	+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	4273	+ [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),
	4274	+#endif
	4275	+#ifdef CONFIG_XFRM
	4276	+ [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),
	4277	+#endif
	4278	+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	4279	+ [TC_SKB_EXT] = SKB_EXT_CHUNKSIZEOF(struct tc_skb_ext),
	4280	+#endif
	4281	+#if IS_ENABLED(CONFIG_MPTCP)
	4282	+ [SKB_EXT_MPTCP] = SKB_EXT_CHUNKSIZEOF(struct mptcp_ext),
	4283	+#endif
	4284	+#if IS_ENABLED(CONFIG_KCOV)
	4285	+ [SKB_EXT_KCOV_HANDLE] = SKB_EXT_CHUNKSIZEOF(u64),
	4286	+#endif
	4287	+};
	4288	+
	4289	+static __always_inline unsigned int skb_ext_total_length(void)
	4290	+{
	4291	+ return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +
	4292	+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
	4293	+ skb_ext_type_len[SKB_EXT_BRIDGE_NF] +
	4294	+#endif
	4295	+#ifdef CONFIG_XFRM
	4296	+ skb_ext_type_len[SKB_EXT_SEC_PATH] +
	4297	+#endif
	4298	+#if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
	4299	+ skb_ext_type_len[TC_SKB_EXT] +
	4300	+#endif
	4301	+#if IS_ENABLED(CONFIG_MPTCP)
	4302	+ skb_ext_type_len[SKB_EXT_MPTCP] +
	4303	+#endif
	4304	+#if IS_ENABLED(CONFIG_KCOV)
	4305	+ skb_ext_type_len[SKB_EXT_KCOV_HANDLE] +
	4306	+#endif
	4307	+ 0;
	4308	+}
	4309	+
	4310	+static void skb_extensions_init(void)
	4311	+{
	4312	+ BUILD_BUG_ON(SKB_EXT_NUM >= 8);
	4313	+ BUILD_BUG_ON(skb_ext_total_length() > 255);
	4314	+
	4315	+ skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",
	4316	+ SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),
	4317	+ 0,
	4318	+ SLAB_HWCACHE_ALIGN\|SLAB_PANIC,
	4319	+ NULL);
	4320	+}
	4321	+#else
	4322	+static void skb_extensions_init(void) {}
	4323	+#endif
4002	4324
4003	4325	void __init skb_init(void)
4004	4326	{
..	..	@@ -4014,6 +4336,7 @@
4014	4336	0,
4015	4337	SLAB_HWCACHE_ALIGN\|SLAB_PANIC,
4016	4338	NULL);
	4339	+ skb_extensions_init();
4017	4340	}
4018	4341
4019	4342	static int
..	..	@@ -4052,7 +4375,7 @@
4052	4375	if (copy > len)
4053	4376	copy = len;
4054	4377	sg_set_page(&sg[elt], skb_frag_page(frag), copy,
4055		- frag->page_offset+offset-start);
	4378	+ skb_frag_off(frag) + offset - start);
4056	4379	elt++;
4057	4380	if (!(len -= copy))
4058	4381	return elt;
..	..	@@ -4169,7 +4492,7 @@
4169	4492	* at the moment even if they are anonymous).
4170	4493	*/
4171	4494	if ((skb_cloned(skb) \|\| skb_shinfo(skb)->nr_frags) &&
4172		- __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
	4495	+ !__pskb_pull_tail(skb, __skb_pagelen(skb)))
4173	4496	return -ENOMEM;
4174	4497
4175	4498	/* Easy case. Most of packets will go this way. */
..	..	@@ -4273,7 +4596,7 @@
4273	4596	int sock_queue_err_skb(struct sock sk, struct sk_buff skb)
4274	4597	{
4275	4598	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
4276		- (unsigned int)sk->sk_rcvbuf)
	4599	+ (unsigned int)READ_ONCE(sk->sk_rcvbuf))
4277	4600	return -ENOMEM;
4278	4601
4279	4602	skb_orphan(skb);
..	..	@@ -4392,7 +4715,7 @@
4392	4715	{
4393	4716	bool ret;
4394	4717
4395		- if (likely(sysctl_tstamp_allow_data \|\| tsonly))
	4718	+ if (likely(READ_ONCE(sysctl_tstamp_allow_data) \|\| tsonly))
4396	4719	return true;
4397	4720
4398	4721	read_lock_bh(&sk->sk_callback_lock);
..	..	@@ -4448,7 +4771,7 @@
4448	4771	if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) &&
4449	4772	sk->sk_protocol == IPPROTO_TCP &&
4450	4773	sk->sk_type == SOCK_STREAM) {
4451		- skb = tcp_get_timestamping_opt_stats(sk);
	4774	+ skb = tcp_get_timestamping_opt_stats(sk, orig_skb);
4452	4775	opt_stats = true;
4453	4776	} else
4454	4777	#endif
..	..	@@ -4565,9 +4888,9 @@
4565	4888	typeof(IPPROTO_IP) proto,
4566	4889	unsigned int off)
4567	4890	{
4568		- switch (proto) {
4569		- int err;
	4891	+ int err;
4570	4892
	4893	+ switch (proto) {
4571	4894	case IPPROTO_TCP:
4572	4895	err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr),
4573	4896	off + MAX_TCP_HDR_LEN);
..	..	@@ -4610,7 +4933,7 @@
4610	4933	if (err < 0)
4611	4934	goto out;
4612	4935
4613		- if (ip_hdr(skb)->frag_off & htons(IP_OFFSET \| IP_MF))
	4936	+ if (ip_is_fragment(ip_hdr(skb)))
4614	4937	fragment = true;
4615	4938
4616	4939	off = ip_hdrlen(skb);
..	..	@@ -4977,13 +5300,13 @@
4977	5300	skb->skb_iif = 0;
4978	5301	skb->ignore_df = 0;
4979	5302	skb_dst_drop(skb);
4980		- secpath_reset(skb);
4981		- nf_reset(skb);
	5303	+ skb_ext_reset(skb);
	5304	+ nf_reset_ct(skb);
4982	5305	nf_reset_trace(skb);
4983	5306
4984	5307	#ifdef CONFIG_NET_SWITCHDEV
4985	5308	skb->offload_fwd_mark = 0;
4986		- skb->offload_mr_fwd_mark = 0;
	5309	+ skb->offload_l3_fwd_mark = 0;
4987	5310	#endif
4988	5311
4989	5312	if (!xnet)
..	..	@@ -5075,6 +5398,8 @@
5075	5398	* - L2+L3+L4+payload size (e.g. sanity check before passing to driver)
5076	5399	*
5077	5400	* This is a helper to do that correctly considering GSO_BY_FRAGS.
	5401	+ *
	5402	+ * @skb: GSO skb
5078	5403	*
5079	5404	* @seg_len: The segmented length (from skb_gso_*_seglen). In the
5080	5405	* GSO_BY_FRAGS case this will be [header sizes + GSO_BY_FRAGS].
..	..	@@ -5261,7 +5586,7 @@
5261	5586	int err;
5262	5587
5263	5588	if (likely(skb_vlan_tag_present(skb))) {
5264		- skb->vlan_tci = 0;
	5589	+ __vlan_hwaccel_clear_tag(skb);
5265	5590	} else {
5266	5591	if (unlikely(!eth_type_vlan(skb->protocol)))
5267	5592	return 0;
..	..	@@ -5313,6 +5638,252 @@
5313	5638	return 0;
5314	5639	}
5315	5640	EXPORT_SYMBOL(skb_vlan_push);
	5641	+
	5642	+/**
	5643	+ * skb_eth_pop() - Drop the Ethernet header at the head of a packet
	5644	+ *
	5645	+ * @skb: Socket buffer to modify
	5646	+ *
	5647	+ * Drop the Ethernet header of @skb.
	5648	+ *
	5649	+ * Expects that skb->data points to the mac header and that no VLAN tags are
	5650	+ * present.
	5651	+ *
	5652	+ * Returns 0 on success, -errno otherwise.
	5653	+ */
	5654	+int skb_eth_pop(struct sk_buff *skb)
	5655	+{
	5656	+ if (!pskb_may_pull(skb, ETH_HLEN) \|\| skb_vlan_tagged(skb) \|\|
	5657	+ skb_network_offset(skb) < ETH_HLEN)
	5658	+ return -EPROTO;
	5659	+
	5660	+ skb_pull_rcsum(skb, ETH_HLEN);
	5661	+ skb_reset_mac_header(skb);
	5662	+ skb_reset_mac_len(skb);
	5663	+
	5664	+ return 0;
	5665	+}
	5666	+EXPORT_SYMBOL(skb_eth_pop);
	5667	+
	5668	+/**
	5669	+ * skb_eth_push() - Add a new Ethernet header at the head of a packet
	5670	+ *
	5671	+ * @skb: Socket buffer to modify
	5672	+ * @dst: Destination MAC address of the new header
	5673	+ * @src: Source MAC address of the new header
	5674	+ *
	5675	+ * Prepend @skb with a new Ethernet header.
	5676	+ *
	5677	+ * Expects that skb->data points to the mac header, which must be empty.
	5678	+ *
	5679	+ * Returns 0 on success, -errno otherwise.
	5680	+ */
	5681	+int skb_eth_push(struct sk_buff skb, const unsigned char dst,
	5682	+ const unsigned char *src)
	5683	+{
	5684	+ struct ethhdr *eth;
	5685	+ int err;
	5686	+
	5687	+ if (skb_network_offset(skb) \|\| skb_vlan_tag_present(skb))
	5688	+ return -EPROTO;
	5689	+
	5690	+ err = skb_cow_head(skb, sizeof(*eth));
	5691	+ if (err < 0)
	5692	+ return err;
	5693	+
	5694	+ skb_push(skb, sizeof(*eth));
	5695	+ skb_reset_mac_header(skb);
	5696	+ skb_reset_mac_len(skb);
	5697	+
	5698	+ eth = eth_hdr(skb);
	5699	+ ether_addr_copy(eth->h_dest, dst);
	5700	+ ether_addr_copy(eth->h_source, src);
	5701	+ eth->h_proto = skb->protocol;
	5702	+
	5703	+ skb_postpush_rcsum(skb, eth, sizeof(*eth));
	5704	+
	5705	+ return 0;
	5706	+}
	5707	+EXPORT_SYMBOL(skb_eth_push);
	5708	+
	5709	+/* Update the ethertype of hdr and the skb csum value if required. */
	5710	+static void skb_mod_eth_type(struct sk_buff skb, struct ethhdr hdr,
	5711	+ __be16 ethertype)
	5712	+{
	5713	+ if (skb->ip_summed == CHECKSUM_COMPLETE) {
	5714	+ __be16 diff[] = { ~hdr->h_proto, ethertype };
	5715	+
	5716	+ skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	5717	+ }
	5718	+
	5719	+ hdr->h_proto = ethertype;
	5720	+}
	5721	+
	5722	+/**
	5723	+ * skb_mpls_push() - push a new MPLS header after mac_len bytes from start of
	5724	+ * the packet
	5725	+ *
	5726	+ * @skb: buffer
	5727	+ * @mpls_lse: MPLS label stack entry to push
	5728	+ * @mpls_proto: ethertype of the new MPLS header (expects 0x8847 or 0x8848)
	5729	+ * @mac_len: length of the MAC header
	5730	+ * @ethernet: flag to indicate if the resulting packet after skb_mpls_push is
	5731	+ * ethernet
	5732	+ *
	5733	+ * Expects skb->data at mac header.
	5734	+ *
	5735	+ * Returns 0 on success, -errno otherwise.
	5736	+ */
	5737	+int skb_mpls_push(struct sk_buff *skb, __be32 mpls_lse, __be16 mpls_proto,
	5738	+ int mac_len, bool ethernet)
	5739	+{
	5740	+ struct mpls_shim_hdr *lse;
	5741	+ int err;
	5742	+
	5743	+ if (unlikely(!eth_p_mpls(mpls_proto)))
	5744	+ return -EINVAL;
	5745	+
	5746	+ /* Networking stack does not allow simultaneous Tunnel and MPLS GSO. */
	5747	+ if (skb->encapsulation)
	5748	+ return -EINVAL;
	5749	+
	5750	+ err = skb_cow_head(skb, MPLS_HLEN);
	5751	+ if (unlikely(err))
	5752	+ return err;
	5753	+
	5754	+ if (!skb->inner_protocol) {
	5755	+ skb_set_inner_network_header(skb, skb_network_offset(skb));
	5756	+ skb_set_inner_protocol(skb, skb->protocol);
	5757	+ }
	5758	+
	5759	+ skb_push(skb, MPLS_HLEN);
	5760	+ memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
	5761	+ mac_len);
	5762	+ skb_reset_mac_header(skb);
	5763	+ skb_set_network_header(skb, mac_len);
	5764	+ skb_reset_mac_len(skb);
	5765	+
	5766	+ lse = mpls_hdr(skb);
	5767	+ lse->label_stack_entry = mpls_lse;
	5768	+ skb_postpush_rcsum(skb, lse, MPLS_HLEN);
	5769	+
	5770	+ if (ethernet && mac_len >= ETH_HLEN)
	5771	+ skb_mod_eth_type(skb, eth_hdr(skb), mpls_proto);
	5772	+ skb->protocol = mpls_proto;
	5773	+
	5774	+ return 0;
	5775	+}
	5776	+EXPORT_SYMBOL_GPL(skb_mpls_push);
	5777	+
	5778	+/**
	5779	+ * skb_mpls_pop() - pop the outermost MPLS header
	5780	+ *
	5781	+ * @skb: buffer
	5782	+ * @next_proto: ethertype of header after popped MPLS header
	5783	+ * @mac_len: length of the MAC header
	5784	+ * @ethernet: flag to indicate if the packet is ethernet
	5785	+ *
	5786	+ * Expects skb->data at mac header.
	5787	+ *
	5788	+ * Returns 0 on success, -errno otherwise.
	5789	+ */
	5790	+int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len,
	5791	+ bool ethernet)
	5792	+{
	5793	+ int err;
	5794	+
	5795	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5796	+ return 0;
	5797	+
	5798	+ err = skb_ensure_writable(skb, mac_len + MPLS_HLEN);
	5799	+ if (unlikely(err))
	5800	+ return err;
	5801	+
	5802	+ skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
	5803	+ memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
	5804	+ mac_len);
	5805	+
	5806	+ __skb_pull(skb, MPLS_HLEN);
	5807	+ skb_reset_mac_header(skb);
	5808	+ skb_set_network_header(skb, mac_len);
	5809	+
	5810	+ if (ethernet && mac_len >= ETH_HLEN) {
	5811	+ struct ethhdr *hdr;
	5812	+
	5813	+ /* use mpls_hdr() to get ethertype to account for VLANs. */
	5814	+ hdr = (struct ethhdr )((void )mpls_hdr(skb) - ETH_HLEN);
	5815	+ skb_mod_eth_type(skb, hdr, next_proto);
	5816	+ }
	5817	+ skb->protocol = next_proto;
	5818	+
	5819	+ return 0;
	5820	+}
	5821	+EXPORT_SYMBOL_GPL(skb_mpls_pop);
	5822	+
	5823	+/**
	5824	+ * skb_mpls_update_lse() - modify outermost MPLS header and update csum
	5825	+ *
	5826	+ * @skb: buffer
	5827	+ * @mpls_lse: new MPLS label stack entry to update to
	5828	+ *
	5829	+ * Expects skb->data at mac header.
	5830	+ *
	5831	+ * Returns 0 on success, -errno otherwise.
	5832	+ */
	5833	+int skb_mpls_update_lse(struct sk_buff *skb, __be32 mpls_lse)
	5834	+{
	5835	+ int err;
	5836	+
	5837	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5838	+ return -EINVAL;
	5839	+
	5840	+ err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
	5841	+ if (unlikely(err))
	5842	+ return err;
	5843	+
	5844	+ if (skb->ip_summed == CHECKSUM_COMPLETE) {
	5845	+ __be32 diff[] = { ~mpls_hdr(skb)->label_stack_entry, mpls_lse };
	5846	+
	5847	+ skb->csum = csum_partial((char *)diff, sizeof(diff), skb->csum);
	5848	+ }
	5849	+
	5850	+ mpls_hdr(skb)->label_stack_entry = mpls_lse;
	5851	+
	5852	+ return 0;
	5853	+}
	5854	+EXPORT_SYMBOL_GPL(skb_mpls_update_lse);
	5855	+
	5856	+/**
	5857	+ * skb_mpls_dec_ttl() - decrement the TTL of the outermost MPLS header
	5858	+ *
	5859	+ * @skb: buffer
	5860	+ *
	5861	+ * Expects skb->data at mac header.
	5862	+ *
	5863	+ * Returns 0 on success, -errno otherwise.
	5864	+ */
	5865	+int skb_mpls_dec_ttl(struct sk_buff *skb)
	5866	+{
	5867	+ u32 lse;
	5868	+ u8 ttl;
	5869	+
	5870	+ if (unlikely(!eth_p_mpls(skb->protocol)))
	5871	+ return -EINVAL;
	5872	+
	5873	+ if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
	5874	+ return -ENOMEM;
	5875	+
	5876	+ lse = be32_to_cpu(mpls_hdr(skb)->label_stack_entry);
	5877	+ ttl = (lse & MPLS_LS_TTL_MASK) >> MPLS_LS_TTL_SHIFT;
	5878	+ if (!--ttl)
	5879	+ return -EINVAL;
	5880	+
	5881	+ lse &= ~MPLS_LS_TTL_MASK;
	5882	+ lse \|= ttl << MPLS_LS_TTL_SHIFT;
	5883	+
	5884	+ return skb_mpls_update_lse(skb, cpu_to_be32(lse));
	5885	+}
	5886	+EXPORT_SYMBOL_GPL(skb_mpls_dec_ttl);
5316	5887
5317	5888	/**
5318	5889	* alloc_skb_with_frags - allocate skb with page frags
..	..	@@ -5436,11 +6007,7 @@
5436	6007	skb->head = data;
5437	6008	skb->data = data;
5438	6009	skb->head_frag = 0;
5439		-#ifdef NET_SKBUFF_DATA_USES_OFFSET
5440		- skb->end = size;
5441		-#else
5442		- skb->end = skb->head + size;
5443		-#endif
	6010	+ skb_set_end_offset(skb, size);
5444	6011	skb_set_tail_pointer(skb, skb_headlen(skb));
5445	6012	skb_headers_offset_update(skb, 0);
5446	6013	skb->cloned = 0;
..	..	@@ -5532,8 +6099,7 @@
5532	6099	size = SKB_WITH_OVERHEAD(ksize(data));
5533	6100
5534	6101	memcpy((struct skb_shared_info *)(data + size),
5535		- skb_shinfo(skb), offsetof(struct skb_shared_info,
5536		- frags[skb_shinfo(skb)->nr_frags]));
	6102	+ skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0]));
5537	6103	if (skb_orphan_frags(skb, gfp_mask)) {
5538	6104	kfree(data);
5539	6105	return -ENOMEM;
..	..	@@ -5554,7 +6120,7 @@
5554	6120	* where splitting is expensive.
5555	6121	* 2. Split is accurately. We make this.
5556	6122	*/
5557		- shinfo->frags[0].page_offset += off - pos;
	6123	+ skb_frag_off_add(&shinfo->frags[0], off - pos);
5558	6124	skb_frag_size_sub(&shinfo->frags[0], off - pos);
5559	6125	}
5560	6126	skb_frag_ref(skb, i);
..	..	@@ -5579,11 +6145,7 @@
5579	6145	skb->head = data;
5580	6146	skb->head_frag = 0;
5581	6147	skb->data = data;
5582		-#ifdef NET_SKBUFF_DATA_USES_OFFSET
5583		- skb->end = size;
5584		-#else
5585		- skb->end = skb->head + size;
5586		-#endif
	6148	+ skb_set_end_offset(skb, size);
5587	6149	skb_reset_tail_pointer(skb);
5588	6150	skb_headers_offset_update(skb, 0);
5589	6151	skb->cloned = 0;
..	..	@@ -5657,4 +6219,181 @@
5657	6219	*/
5658	6220	skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
5659	6221	}
5660		-EXPORT_SYMBOL_GPL(skb_condense);
	6222	+
	6223	+#ifdef CONFIG_SKB_EXTENSIONS
	6224	+static void skb_ext_get_ptr(struct skb_ext ext, enum skb_ext_id id)
	6225	+{
	6226	+ return (void )ext + (ext->offset[id] SKB_EXT_ALIGN_VALUE);
	6227	+}
	6228	+
	6229	+/**
	6230	+ * __skb_ext_alloc - allocate a new skb extensions storage
	6231	+ *
	6232	+ * @flags: See kmalloc().
	6233	+ *
	6234	+ * Returns the newly allocated pointer. The pointer can later attached to a
	6235	+ * skb via __skb_ext_set().
	6236	+ * Note: caller must handle the skb_ext as an opaque data.
	6237	+ */
	6238	+struct skb_ext *__skb_ext_alloc(gfp_t flags)
	6239	+{
	6240	+ struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, flags);
	6241	+
	6242	+ if (new) {
	6243	+ memset(new->offset, 0, sizeof(new->offset));
	6244	+ refcount_set(&new->refcnt, 1);
	6245	+ }
	6246	+
	6247	+ return new;
	6248	+}
	6249	+
	6250	+static struct skb_ext skb_ext_maybe_cow(struct skb_ext old,
	6251	+ unsigned int old_active)
	6252	+{
	6253	+ struct skb_ext *new;
	6254	+
	6255	+ if (refcount_read(&old->refcnt) == 1)
	6256	+ return old;
	6257	+
	6258	+ new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);
	6259	+ if (!new)
	6260	+ return NULL;
	6261	+
	6262	+ memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);
	6263	+ refcount_set(&new->refcnt, 1);
	6264	+
	6265	+#ifdef CONFIG_XFRM
	6266	+ if (old_active & (1 << SKB_EXT_SEC_PATH)) {
	6267	+ struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);
	6268	+ unsigned int i;
	6269	+
	6270	+ for (i = 0; i < sp->len; i++)
	6271	+ xfrm_state_hold(sp->xvec[i]);
	6272	+ }
	6273	+#endif
	6274	+ __skb_ext_put(old);
	6275	+ return new;
	6276	+}
	6277	+
	6278	+/**
	6279	+ * __skb_ext_set - attach the specified extension storage to this skb
	6280	+ * @skb: buffer
	6281	+ * @id: extension id
	6282	+ * @ext: extension storage previously allocated via __skb_ext_alloc()
	6283	+ *
	6284	+ * Existing extensions, if any, are cleared.
	6285	+ *
	6286	+ * Returns the pointer to the extension.
	6287	+ */
	6288	+void __skb_ext_set(struct sk_buff skb, enum skb_ext_id id,
	6289	+ struct skb_ext *ext)
	6290	+{
	6291	+ unsigned int newlen, newoff = SKB_EXT_CHUNKSIZEOF(*ext);
	6292	+
	6293	+ skb_ext_put(skb);
	6294	+ newlen = newoff + skb_ext_type_len[id];
	6295	+ ext->chunks = newlen;
	6296	+ ext->offset[id] = newoff;
	6297	+ skb->extensions = ext;
	6298	+ skb->active_extensions = 1 << id;
	6299	+ return skb_ext_get_ptr(ext, id);
	6300	+}
	6301	+
	6302	+/**
	6303	+ * skb_ext_add - allocate space for given extension, COW if needed
	6304	+ * @skb: buffer
	6305	+ * @id: extension to allocate space for
	6306	+ *
	6307	+ * Allocates enough space for the given extension.
	6308	+ * If the extension is already present, a pointer to that extension
	6309	+ * is returned.
	6310	+ *
	6311	+ * If the skb was cloned, COW applies and the returned memory can be
	6312	+ * modified without changing the extension space of clones buffers.
	6313	+ *
	6314	+ * Returns pointer to the extension or NULL on allocation failure.
	6315	+ */
	6316	+void skb_ext_add(struct sk_buff skb, enum skb_ext_id id)
	6317	+{
	6318	+ struct skb_ext new, old = NULL;
	6319	+ unsigned int newlen, newoff;
	6320	+
	6321	+ if (skb->active_extensions) {
	6322	+ old = skb->extensions;
	6323	+
	6324	+ new = skb_ext_maybe_cow(old, skb->active_extensions);
	6325	+ if (!new)
	6326	+ return NULL;
	6327	+
	6328	+ if (__skb_ext_exist(new, id))
	6329	+ goto set_active;
	6330	+
	6331	+ newoff = new->chunks;
	6332	+ } else {
	6333	+ newoff = SKB_EXT_CHUNKSIZEOF(*new);
	6334	+
	6335	+ new = __skb_ext_alloc(GFP_ATOMIC);
	6336	+ if (!new)
	6337	+ return NULL;
	6338	+ }
	6339	+
	6340	+ newlen = newoff + skb_ext_type_len[id];
	6341	+ new->chunks = newlen;
	6342	+ new->offset[id] = newoff;
	6343	+set_active:
	6344	+ skb->extensions = new;
	6345	+ skb->active_extensions \|= 1 << id;
	6346	+ return skb_ext_get_ptr(new, id);
	6347	+}
	6348	+EXPORT_SYMBOL(skb_ext_add);
	6349	+
	6350	+#ifdef CONFIG_XFRM
	6351	+static void skb_ext_put_sp(struct sec_path *sp)
	6352	+{
	6353	+ unsigned int i;
	6354	+
	6355	+ for (i = 0; i < sp->len; i++)
	6356	+ xfrm_state_put(sp->xvec[i]);
	6357	+}
	6358	+#endif
	6359	+
	6360	+void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)
	6361	+{
	6362	+ struct skb_ext *ext = skb->extensions;
	6363	+
	6364	+ skb->active_extensions &= ~(1 << id);
	6365	+ if (skb->active_extensions == 0) {
	6366	+ skb->extensions = NULL;
	6367	+ __skb_ext_put(ext);
	6368	+#ifdef CONFIG_XFRM
	6369	+ } else if (id == SKB_EXT_SEC_PATH &&
	6370	+ refcount_read(&ext->refcnt) == 1) {
	6371	+ struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);
	6372	+
	6373	+ skb_ext_put_sp(sp);
	6374	+ sp->len = 0;
	6375	+#endif
	6376	+ }
	6377	+}
	6378	+EXPORT_SYMBOL(__skb_ext_del);
	6379	+
	6380	+void __skb_ext_put(struct skb_ext *ext)
	6381	+{
	6382	+ /* If this is last clone, nothing can increment
	6383	+ * it after check passes. Avoids one atomic op.
	6384	+ */
	6385	+ if (refcount_read(&ext->refcnt) == 1)
	6386	+ goto free_now;
	6387	+
	6388	+ if (!refcount_dec_and_test(&ext->refcnt))
	6389	+ return;
	6390	+free_now:
	6391	+#ifdef CONFIG_XFRM
	6392	+ if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))
	6393	+ skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));
	6394	+#endif
	6395	+
	6396	+ kmem_cache_free(skbuff_ext_cache, ext);
	6397	+}
	6398	+EXPORT_SYMBOL(__skb_ext_put);
	6399	+#endif /* CONFIG_SKB_EXTENSIONS */