~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* xfrm_state.c
3	4	*
..	..	@@ -26,6 +27,8 @@
26	27	#include <linux/interrupt.h>
27	28	#include <linux/kernel.h>
28	29
	30	+#include <crypto/aead.h>
	31	+
29	32	#include "xfrm_hash.h"
30	33
31	34	#define xfrm_state_deref_prot(table, net) \
..	..	@@ -41,7 +44,6 @@
41	44	*/
42	45
43	46	static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
44		-static __read_mostly seqcount_t xfrm_state_hash_generation = SEQCNT_ZERO(xfrm_state_hash_generation);
45	47	static struct kmem_cache *xfrm_state_cache __ro_after_init;
46	48
47	49	static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
..	..	@@ -137,7 +139,7 @@
137	139	}
138	140
139	141	spin_lock_bh(&net->xfrm.xfrm_state_lock);
140		- write_seqcount_begin(&xfrm_state_hash_generation);
	142	+ write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
141	143
142	144	nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
143	145	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
..	..	@@ -153,7 +155,7 @@
153	155	rcu_assign_pointer(net->xfrm.state_byspi, nspi);
154	156	net->xfrm.state_hmask = nhashmask;
155	157
156		- write_seqcount_end(&xfrm_state_hash_generation);
	158	+ write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
157	159	spin_unlock_bh(&net->xfrm.xfrm_state_lock);
158	160
159	161	osize = (ohashmask + 1) * sizeof(struct hlist_head);
..	..	@@ -173,66 +175,135 @@
173	175	int __xfrm_state_delete(struct xfrm_state *x);
174	176
175	177	int km_query(struct xfrm_state x, struct xfrm_tmpl t, struct xfrm_policy *pol);
176		-bool km_is_alive(const struct km_event *c);
	178	+static bool km_is_alive(const struct km_event *c);
177	179	void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
178	180
179		-static DEFINE_SPINLOCK(xfrm_type_lock);
180	181	int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
181	182	{
182	183	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
183		- const struct xfrm_type **typemap;
184	184	int err = 0;
185	185
186		- if (unlikely(afinfo == NULL))
	186	+ if (!afinfo)
187	187	return -EAFNOSUPPORT;
188		- typemap = afinfo->type_map;
189		- spin_lock_bh(&xfrm_type_lock);
190	188
191		- if (likely(typemap[type->proto] == NULL))
192		- typemap[type->proto] = type;
193		- else
194		- err = -EEXIST;
195		- spin_unlock_bh(&xfrm_type_lock);
	189	+#define X(afi, T, name) do { \
	190	+ WARN_ON((afi)->type_ ## name); \
	191	+ (afi)->type_ ## name = (T); \
	192	+ } while (0)
	193	+
	194	+ switch (type->proto) {
	195	+ case IPPROTO_COMP:
	196	+ X(afinfo, type, comp);
	197	+ break;
	198	+ case IPPROTO_AH:
	199	+ X(afinfo, type, ah);
	200	+ break;
	201	+ case IPPROTO_ESP:
	202	+ X(afinfo, type, esp);
	203	+ break;
	204	+ case IPPROTO_IPIP:
	205	+ X(afinfo, type, ipip);
	206	+ break;
	207	+ case IPPROTO_DSTOPTS:
	208	+ X(afinfo, type, dstopts);
	209	+ break;
	210	+ case IPPROTO_ROUTING:
	211	+ X(afinfo, type, routing);
	212	+ break;
	213	+ case IPPROTO_IPV6:
	214	+ X(afinfo, type, ipip6);
	215	+ break;
	216	+ default:
	217	+ WARN_ON(1);
	218	+ err = -EPROTONOSUPPORT;
	219	+ break;
	220	+ }
	221	+#undef X
196	222	rcu_read_unlock();
197	223	return err;
198	224	}
199	225	EXPORT_SYMBOL(xfrm_register_type);
200	226
201		-int xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
	227	+void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
202	228	{
203	229	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
204		- const struct xfrm_type **typemap;
205		- int err = 0;
206	230
207	231	if (unlikely(afinfo == NULL))
208		- return -EAFNOSUPPORT;
209		- typemap = afinfo->type_map;
210		- spin_lock_bh(&xfrm_type_lock);
	232	+ return;
211	233
212		- if (unlikely(typemap[type->proto] != type))
213		- err = -ENOENT;
214		- else
215		- typemap[type->proto] = NULL;
216		- spin_unlock_bh(&xfrm_type_lock);
	234	+#define X(afi, T, name) do { \
	235	+ WARN_ON((afi)->type_ ## name != (T)); \
	236	+ (afi)->type_ ## name = NULL; \
	237	+ } while (0)
	238	+
	239	+ switch (type->proto) {
	240	+ case IPPROTO_COMP:
	241	+ X(afinfo, type, comp);
	242	+ break;
	243	+ case IPPROTO_AH:
	244	+ X(afinfo, type, ah);
	245	+ break;
	246	+ case IPPROTO_ESP:
	247	+ X(afinfo, type, esp);
	248	+ break;
	249	+ case IPPROTO_IPIP:
	250	+ X(afinfo, type, ipip);
	251	+ break;
	252	+ case IPPROTO_DSTOPTS:
	253	+ X(afinfo, type, dstopts);
	254	+ break;
	255	+ case IPPROTO_ROUTING:
	256	+ X(afinfo, type, routing);
	257	+ break;
	258	+ case IPPROTO_IPV6:
	259	+ X(afinfo, type, ipip6);
	260	+ break;
	261	+ default:
	262	+ WARN_ON(1);
	263	+ break;
	264	+ }
	265	+#undef X
217	266	rcu_read_unlock();
218		- return err;
219	267	}
220	268	EXPORT_SYMBOL(xfrm_unregister_type);
221	269
222	270	static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
223	271	{
	272	+ const struct xfrm_type *type = NULL;
224	273	struct xfrm_state_afinfo *afinfo;
225		- const struct xfrm_type **typemap;
226		- const struct xfrm_type *type;
227	274	int modload_attempted = 0;
228	275
229	276	retry:
230	277	afinfo = xfrm_state_get_afinfo(family);
231	278	if (unlikely(afinfo == NULL))
232	279	return NULL;
233		- typemap = afinfo->type_map;
234	280
235		- type = READ_ONCE(typemap[proto]);
	281	+ switch (proto) {
	282	+ case IPPROTO_COMP:
	283	+ type = afinfo->type_comp;
	284	+ break;
	285	+ case IPPROTO_AH:
	286	+ type = afinfo->type_ah;
	287	+ break;
	288	+ case IPPROTO_ESP:
	289	+ type = afinfo->type_esp;
	290	+ break;
	291	+ case IPPROTO_IPIP:
	292	+ type = afinfo->type_ipip;
	293	+ break;
	294	+ case IPPROTO_DSTOPTS:
	295	+ type = afinfo->type_dstopts;
	296	+ break;
	297	+ case IPPROTO_ROUTING:
	298	+ type = afinfo->type_routing;
	299	+ break;
	300	+ case IPPROTO_IPV6:
	301	+ type = afinfo->type_ipip6;
	302	+ break;
	303	+ default:
	304	+ break;
	305	+ }
	306	+
236	307	if (unlikely(type && !try_module_get(type->owner)))
237	308	type = NULL;
238	309
..	..	@@ -252,65 +323,71 @@
252	323	module_put(type->owner);
253	324	}
254	325
255		-static DEFINE_SPINLOCK(xfrm_type_offload_lock);
256	326	int xfrm_register_type_offload(const struct xfrm_type_offload *type,
257	327	unsigned short family)
258	328	{
259	329	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
260		- const struct xfrm_type_offload **typemap;
261	330	int err = 0;
262	331
263	332	if (unlikely(afinfo == NULL))
264	333	return -EAFNOSUPPORT;
265		- typemap = afinfo->type_offload_map;
266		- spin_lock_bh(&xfrm_type_offload_lock);
267	334
268		- if (likely(typemap[type->proto] == NULL))
269		- typemap[type->proto] = type;
270		- else
271		- err = -EEXIST;
272		- spin_unlock_bh(&xfrm_type_offload_lock);
	335	+ switch (type->proto) {
	336	+ case IPPROTO_ESP:
	337	+ WARN_ON(afinfo->type_offload_esp);
	338	+ afinfo->type_offload_esp = type;
	339	+ break;
	340	+ default:
	341	+ WARN_ON(1);
	342	+ err = -EPROTONOSUPPORT;
	343	+ break;
	344	+ }
	345	+
273	346	rcu_read_unlock();
274	347	return err;
275	348	}
276	349	EXPORT_SYMBOL(xfrm_register_type_offload);
277	350
278		-int xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
279		- unsigned short family)
	351	+void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
	352	+ unsigned short family)
280	353	{
281	354	struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
282		- const struct xfrm_type_offload **typemap;
283		- int err = 0;
284	355
285	356	if (unlikely(afinfo == NULL))
286		- return -EAFNOSUPPORT;
287		- typemap = afinfo->type_offload_map;
288		- spin_lock_bh(&xfrm_type_offload_lock);
	357	+ return;
289	358
290		- if (unlikely(typemap[type->proto] != type))
291		- err = -ENOENT;
292		- else
293		- typemap[type->proto] = NULL;
294		- spin_unlock_bh(&xfrm_type_offload_lock);
	359	+ switch (type->proto) {
	360	+ case IPPROTO_ESP:
	361	+ WARN_ON(afinfo->type_offload_esp != type);
	362	+ afinfo->type_offload_esp = NULL;
	363	+ break;
	364	+ default:
	365	+ WARN_ON(1);
	366	+ break;
	367	+ }
295	368	rcu_read_unlock();
296		- return err;
297	369	}
298	370	EXPORT_SYMBOL(xfrm_unregister_type_offload);
299	371
300	372	static const struct xfrm_type_offload *
301	373	xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
302	374	{
	375	+ const struct xfrm_type_offload *type = NULL;
303	376	struct xfrm_state_afinfo *afinfo;
304		- const struct xfrm_type_offload **typemap;
305		- const struct xfrm_type_offload *type;
306	377
307	378	retry:
308	379	afinfo = xfrm_state_get_afinfo(family);
309	380	if (unlikely(afinfo == NULL))
310	381	return NULL;
311		- typemap = afinfo->type_offload_map;
312	382
313		- type = typemap[proto];
	383	+ switch (proto) {
	384	+ case IPPROTO_ESP:
	385	+ type = afinfo->type_offload_esp;
	386	+ break;
	387	+ default:
	388	+ break;
	389	+ }
	390	+
314	391	if ((type && !try_module_get(type->owner)))
315	392	type = NULL;
316	393
..	..	@@ -330,100 +407,67 @@
330	407	module_put(type->owner);
331	408	}
332	409
333		-static DEFINE_SPINLOCK(xfrm_mode_lock);
334		-int xfrm_register_mode(struct xfrm_mode *mode, int family)
	410	+static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
	411	+ [XFRM_MODE_BEET] = {
	412	+ .encap = XFRM_MODE_BEET,
	413	+ .flags = XFRM_MODE_FLAG_TUNNEL,
	414	+ .family = AF_INET,
	415	+ },
	416	+ [XFRM_MODE_TRANSPORT] = {
	417	+ .encap = XFRM_MODE_TRANSPORT,
	418	+ .family = AF_INET,
	419	+ },
	420	+ [XFRM_MODE_TUNNEL] = {
	421	+ .encap = XFRM_MODE_TUNNEL,
	422	+ .flags = XFRM_MODE_FLAG_TUNNEL,
	423	+ .family = AF_INET,
	424	+ },
	425	+};
	426	+
	427	+static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
	428	+ [XFRM_MODE_BEET] = {
	429	+ .encap = XFRM_MODE_BEET,
	430	+ .flags = XFRM_MODE_FLAG_TUNNEL,
	431	+ .family = AF_INET6,
	432	+ },
	433	+ [XFRM_MODE_ROUTEOPTIMIZATION] = {
	434	+ .encap = XFRM_MODE_ROUTEOPTIMIZATION,
	435	+ .family = AF_INET6,
	436	+ },
	437	+ [XFRM_MODE_TRANSPORT] = {
	438	+ .encap = XFRM_MODE_TRANSPORT,
	439	+ .family = AF_INET6,
	440	+ },
	441	+ [XFRM_MODE_TUNNEL] = {
	442	+ .encap = XFRM_MODE_TUNNEL,
	443	+ .flags = XFRM_MODE_FLAG_TUNNEL,
	444	+ .family = AF_INET6,
	445	+ },
	446	+};
	447	+
	448	+static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
335	449	{
336		- struct xfrm_state_afinfo *afinfo;
337		- struct xfrm_mode **modemap;
338		- int err;
339		-
340		- if (unlikely(mode->encap >= XFRM_MODE_MAX))
341		- return -EINVAL;
342		-
343		- afinfo = xfrm_state_get_afinfo(family);
344		- if (unlikely(afinfo == NULL))
345		- return -EAFNOSUPPORT;
346		-
347		- err = -EEXIST;
348		- modemap = afinfo->mode_map;
349		- spin_lock_bh(&xfrm_mode_lock);
350		- if (modemap[mode->encap])
351		- goto out;
352		-
353		- err = -ENOENT;
354		- if (!try_module_get(afinfo->owner))
355		- goto out;
356		-
357		- mode->afinfo = afinfo;
358		- modemap[mode->encap] = mode;
359		- err = 0;
360		-
361		-out:
362		- spin_unlock_bh(&xfrm_mode_lock);
363		- rcu_read_unlock();
364		- return err;
365		-}
366		-EXPORT_SYMBOL(xfrm_register_mode);
367		-
368		-int xfrm_unregister_mode(struct xfrm_mode *mode, int family)
369		-{
370		- struct xfrm_state_afinfo *afinfo;
371		- struct xfrm_mode **modemap;
372		- int err;
373		-
374		- if (unlikely(mode->encap >= XFRM_MODE_MAX))
375		- return -EINVAL;
376		-
377		- afinfo = xfrm_state_get_afinfo(family);
378		- if (unlikely(afinfo == NULL))
379		- return -EAFNOSUPPORT;
380		-
381		- err = -ENOENT;
382		- modemap = afinfo->mode_map;
383		- spin_lock_bh(&xfrm_mode_lock);
384		- if (likely(modemap[mode->encap] == mode)) {
385		- modemap[mode->encap] = NULL;
386		- module_put(mode->afinfo->owner);
387		- err = 0;
388		- }
389		-
390		- spin_unlock_bh(&xfrm_mode_lock);
391		- rcu_read_unlock();
392		- return err;
393		-}
394		-EXPORT_SYMBOL(xfrm_unregister_mode);
395		-
396		-static struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
397		-{
398		- struct xfrm_state_afinfo *afinfo;
399		- struct xfrm_mode *mode;
400		- int modload_attempted = 0;
	450	+ const struct xfrm_mode *mode;
401	451
402	452	if (unlikely(encap >= XFRM_MODE_MAX))
403	453	return NULL;
404	454
405		-retry:
406		- afinfo = xfrm_state_get_afinfo(family);
407		- if (unlikely(afinfo == NULL))
408		- return NULL;
409		-
410		- mode = READ_ONCE(afinfo->mode_map[encap]);
411		- if (unlikely(mode && !try_module_get(mode->owner)))
412		- mode = NULL;
413		-
414		- rcu_read_unlock();
415		- if (!mode && !modload_attempted) {
416		- request_module("xfrm-mode-%d-%d", family, encap);
417		- modload_attempted = 1;
418		- goto retry;
	455	+ switch (family) {
	456	+ case AF_INET:
	457	+ mode = &xfrm4_mode_map[encap];
	458	+ if (mode->family == family)
	459	+ return mode;
	460	+ break;
	461	+ case AF_INET6:
	462	+ mode = &xfrm6_mode_map[encap];
	463	+ if (mode->family == family)
	464	+ return mode;
	465	+ break;
	466	+ default:
	467	+ break;
419	468	}
420	469
421		- return mode;
422		-}
423		-
424		-static void xfrm_put_mode(struct xfrm_mode *mode)
425		-{
426		- module_put(mode->owner);
	470	+ return NULL;
427	471	}
428	472
429	473	void xfrm_state_free(struct xfrm_state *x)
..	..	@@ -434,7 +478,7 @@
434	478
435	479	static void ___xfrm_state_destroy(struct xfrm_state *x)
436	480	{
437		- tasklet_hrtimer_cancel(&x->mtimer);
	481	+ hrtimer_cancel(&x->mtimer);
438	482	del_timer_sync(&x->rtimer);
439	483	kfree(x->aead);
440	484	kfree(x->aalg);
..	..	@@ -444,12 +488,6 @@
444	488	kfree(x->coaddr);
445	489	kfree(x->replay_esn);
446	490	kfree(x->preplay_esn);
447		- if (x->inner_mode)
448		- xfrm_put_mode(x->inner_mode);
449		- if (x->inner_mode_iaf)
450		- xfrm_put_mode(x->inner_mode_iaf);
451		- if (x->outer_mode)
452		- xfrm_put_mode(x->outer_mode);
453	491	if (x->type_offload)
454	492	xfrm_put_type_offload(x->type_offload);
455	493	if (x->type) {
..	..	@@ -481,8 +519,8 @@
481	519
482	520	static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
483	521	{
484		- struct tasklet_hrtimer *thr = container_of(me, struct tasklet_hrtimer, timer);
485		- struct xfrm_state *x = container_of(thr, struct xfrm_state, mtimer);
	522	+ struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
	523	+ enum hrtimer_restart ret = HRTIMER_NORESTART;
486	524	time64_t now = ktime_get_real_seconds();
487	525	time64_t next = TIME64_MAX;
488	526	int warn = 0;
..	..	@@ -546,7 +584,8 @@
546	584	km_state_expired(x, 0, 0);
547	585	resched:
548	586	if (next != TIME64_MAX) {
549		- tasklet_hrtimer_start(&x->mtimer, ktime_set(next, 0), HRTIMER_MODE_REL);
	587	+ hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));
	588	+ ret = HRTIMER_RESTART;
550	589	}
551	590
552	591	goto out;
..	..	@@ -563,7 +602,7 @@
563	602
564	603	out:
565	604	spin_unlock(&x->lock);
566		- return HRTIMER_NORESTART;
	605	+ return ret;
567	606	}
568	607
569	608	static void xfrm_replay_timer_handler(struct timer_list *t);
..	..	@@ -572,7 +611,7 @@
572	611	{
573	612	struct xfrm_state *x;
574	613
575		- x = kmem_cache_alloc(xfrm_state_cache, GFP_ATOMIC \| __GFP_ZERO);
	614	+ x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);
576	615
577	616	if (x) {
578	617	write_pnet(&x->xs_net, net);
..	..	@@ -582,8 +621,8 @@
582	621	INIT_HLIST_NODE(&x->bydst);
583	622	INIT_HLIST_NODE(&x->bysrc);
584	623	INIT_HLIST_NODE(&x->byspi);
585		- tasklet_hrtimer_init(&x->mtimer, xfrm_timer_handler,
586		- CLOCK_BOOTTIME, HRTIMER_MODE_ABS);
	624	+ hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
	625	+ x->mtimer.function = xfrm_timer_handler;
587	626	timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
588	627	x->curlft.add_time = ktime_get_real_seconds();
589	628	x->lft.soft_byte_limit = XFRM_INF;
..	..	@@ -592,8 +631,6 @@
592	631	x->lft.hard_packet_limit = XFRM_INF;
593	632	x->replay_maxage = 0;
594	633	x->replay_maxdiff = 0;
595		- x->inner_mode = NULL;
596		- x->inner_mode_iaf = NULL;
597	634	spin_lock_init(&x->lock);
598	635	}
599	636	return x;
..	..	@@ -631,6 +668,9 @@
631	668	hlist_del_rcu(&x->byspi);
632	669	net->xfrm.state_num--;
633	670	spin_unlock(&net->xfrm.xfrm_state_lock);
	671	+
	672	+ if (x->encap_sk)
	673	+ sock_put(rcu_dereference_raw(x->encap_sk));
634	674
635	675	xfrm_dev_state_delete(x);
636	676
..	..	@@ -811,24 +851,79 @@
811	851	EXPORT_SYMBOL(xfrm_sad_getinfo);
812	852
813	853	static void
	854	+__xfrm4_init_tempsel(struct xfrm_selector sel, const struct flowi fl)
	855	+{
	856	+ const struct flowi4 *fl4 = &fl->u.ip4;
	857	+
	858	+ sel->daddr.a4 = fl4->daddr;
	859	+ sel->saddr.a4 = fl4->saddr;
	860	+ sel->dport = xfrm_flowi_dport(fl, &fl4->uli);
	861	+ sel->dport_mask = htons(0xffff);
	862	+ sel->sport = xfrm_flowi_sport(fl, &fl4->uli);
	863	+ sel->sport_mask = htons(0xffff);
	864	+ sel->family = AF_INET;
	865	+ sel->prefixlen_d = 32;
	866	+ sel->prefixlen_s = 32;
	867	+ sel->proto = fl4->flowi4_proto;
	868	+ sel->ifindex = fl4->flowi4_oif;
	869	+}
	870	+
	871	+static void
	872	+__xfrm6_init_tempsel(struct xfrm_selector sel, const struct flowi fl)
	873	+{
	874	+ const struct flowi6 *fl6 = &fl->u.ip6;
	875	+
	876	+ /* Initialize temporary selector matching only to current session. */
	877	+ (struct in6_addr )&sel->daddr = fl6->daddr;
	878	+ (struct in6_addr )&sel->saddr = fl6->saddr;
	879	+ sel->dport = xfrm_flowi_dport(fl, &fl6->uli);
	880	+ sel->dport_mask = htons(0xffff);
	881	+ sel->sport = xfrm_flowi_sport(fl, &fl6->uli);
	882	+ sel->sport_mask = htons(0xffff);
	883	+ sel->family = AF_INET6;
	884	+ sel->prefixlen_d = 128;
	885	+ sel->prefixlen_s = 128;
	886	+ sel->proto = fl6->flowi6_proto;
	887	+ sel->ifindex = fl6->flowi6_oif;
	888	+}
	889	+
	890	+static void
814	891	xfrm_init_tempstate(struct xfrm_state x, const struct flowi fl,
815	892	const struct xfrm_tmpl *tmpl,
816	893	const xfrm_address_t daddr, const xfrm_address_t saddr,
817	894	unsigned short family)
818	895	{
819		- struct xfrm_state_afinfo *afinfo = xfrm_state_afinfo_get_rcu(family);
820		-
821		- if (!afinfo)
822		- return;
823		-
824		- afinfo->init_tempsel(&x->sel, fl);
825		-
826		- if (family != tmpl->encap_family) {
827		- afinfo = xfrm_state_afinfo_get_rcu(tmpl->encap_family);
828		- if (!afinfo)
829		- return;
	896	+ switch (family) {
	897	+ case AF_INET:
	898	+ __xfrm4_init_tempsel(&x->sel, fl);
	899	+ break;
	900	+ case AF_INET6:
	901	+ __xfrm6_init_tempsel(&x->sel, fl);
	902	+ break;
830	903	}
831		- afinfo->init_temprop(x, tmpl, daddr, saddr);
	904	+
	905	+ x->id = tmpl->id;
	906	+
	907	+ switch (tmpl->encap_family) {
	908	+ case AF_INET:
	909	+ if (x->id.daddr.a4 == 0)
	910	+ x->id.daddr.a4 = daddr->a4;
	911	+ x->props.saddr = tmpl->saddr;
	912	+ if (x->props.saddr.a4 == 0)
	913	+ x->props.saddr.a4 = saddr->a4;
	914	+ break;
	915	+ case AF_INET6:
	916	+ if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))
	917	+ memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
	918	+ memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
	919	+ if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))
	920	+ memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
	921	+ break;
	922	+ }
	923	+
	924	+ x->props.mode = tmpl->mode;
	925	+ x->props.reqid = tmpl->reqid;
	926	+ x->props.family = tmpl->encap_family;
832	927	}
833	928
834	929	static struct xfrm_state __xfrm_state_lookup(struct net net, u32 mark,
..	..	@@ -925,7 +1020,8 @@
925	1020	if ((x->sel.family &&
926	1021	(x->sel.family != family \|\|
927	1022	!xfrm_selector_match(&x->sel, fl, family))) \|\|
928		- !security_xfrm_state_pol_flow_match(x, pol, fl))
	1023	+ !security_xfrm_state_pol_flow_match(x, pol,
	1024	+ &fl->u.__fl_common))
929	1025	return;
930	1026
931	1027	if (!*best \|\|
..	..	@@ -940,7 +1036,8 @@
940	1036	if ((!x->sel.family \|\|
941	1037	(x->sel.family == family &&
942	1038	xfrm_selector_match(&x->sel, fl, family))) &&
943		- security_xfrm_state_pol_flow_match(x, pol, fl))
	1039	+ security_xfrm_state_pol_flow_match(x, pol,
	1040	+ &fl->u.__fl_common))
944	1041	*error = -ESRCH;
945	1042	}
946	1043	}
..	..	@@ -965,7 +1062,7 @@
965	1062
966	1063	to_put = NULL;
967	1064
968		- sequence = read_seqcount_begin(&xfrm_state_hash_generation);
	1065	+ sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
969	1066
970	1067	rcu_read_lock();
971	1068	h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
..	..	@@ -1052,7 +1149,9 @@
1052	1149	hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
1053	1150	}
1054	1151	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1055		- tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);
	1152	+ hrtimer_start(&x->mtimer,
	1153	+ ktime_set(net->xfrm.sysctl_acq_expires, 0),
	1154	+ HRTIMER_MODE_REL_SOFT);
1056	1155	net->xfrm.state_num++;
1057	1156	xfrm_hash_grow_check(net, x->bydst.next != NULL);
1058	1157	spin_unlock_bh(&net->xfrm.xfrm_state_lock);
..	..	@@ -1076,7 +1175,7 @@
1076	1175	if (to_put)
1077	1176	xfrm_state_put(to_put);
1078	1177
1079		- if (read_seqcount_retry(&xfrm_state_hash_generation, sequence)) {
	1178	+ if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1080	1179	*err = -EAGAIN;
1081	1180	if (x) {
1082	1181	xfrm_state_put(x);
..	..	@@ -1164,7 +1263,7 @@
1164	1263	hlist_add_head_rcu(&x->byspi, net->xfrm.state_byspi + h);
1165	1264	}
1166	1265
1167		- tasklet_hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL);
	1266	+ hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
1168	1267	if (x->replay_maxage)
1169	1268	mod_timer(&x->rtimer, jiffies + x->replay_maxage);
1170	1269
..	..	@@ -1271,7 +1370,9 @@
1271	1370	x->mark.m = m->m;
1272	1371	x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1273	1372	xfrm_state_hold(x);
1274		- tasklet_hrtimer_start(&x->mtimer, ktime_set(net->xfrm.sysctl_acq_expires, 0), HRTIMER_MODE_REL);
	1373	+ hrtimer_start(&x->mtimer,
	1374	+ ktime_set(net->xfrm.sysctl_acq_expires, 0),
	1375	+ HRTIMER_MODE_REL_SOFT);
1275	1376	list_add(&x->km.all, &net->xfrm.state_all);
1276	1377	hlist_add_head_rcu(&x->bydst, net->xfrm.state_bydst + h);
1277	1378	h = xfrm_src_hash(net, daddr, saddr, family);
..	..	@@ -1443,9 +1544,6 @@
1443	1544	memcpy(&x->mark, &orig->mark, sizeof(x->mark));
1444	1545	memcpy(&x->props.smark, &orig->props.smark, sizeof(x->props.smark));
1445	1546
1446		- if (xfrm_init_state(x) < 0)
1447		- goto error;
1448		-
1449	1547	x->props.flags = orig->props.flags;
1450	1548	x->props.extra_flags = orig->props.extra_flags;
1451	1549
..	..	@@ -1458,6 +1556,7 @@
1458	1556	x->km.seq = orig->km.seq;
1459	1557	x->replay = orig->replay;
1460	1558	x->preplay = orig->preplay;
	1559	+ x->lastused = orig->lastused;
1461	1560
1462	1561	return x;
1463	1562
..	..	@@ -1467,7 +1566,8 @@
1467	1566	return NULL;
1468	1567	}
1469	1568
1470		-struct xfrm_state xfrm_migrate_state_find(struct xfrm_migrate m, struct net *net)
	1569	+struct xfrm_state xfrm_migrate_state_find(struct xfrm_migrate m, struct net *net,
	1570	+ u32 if_id)
1471	1571	{
1472	1572	unsigned int h;
1473	1573	struct xfrm_state *x = NULL;
..	..	@@ -1483,6 +1583,8 @@
1483	1583	continue;
1484	1584	if (m->reqid && x->props.reqid != m->reqid)
1485	1585	continue;
	1586	+ if (if_id != 0 && x->if_id != if_id)
	1587	+ continue;
1486	1588	if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1487	1589	m->old_family) \|\|
1488	1590	!xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
..	..	@@ -1497,6 +1599,8 @@
1497	1599	hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {
1498	1600	if (x->props.mode != m->mode \|\|
1499	1601	x->id.proto != m->proto)
	1602	+ continue;
	1603	+ if (if_id != 0 && x->if_id != if_id)
1500	1604	continue;
1501	1605	if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
1502	1606	m->old_family) \|\|
..	..	@@ -1523,6 +1627,11 @@
1523	1627	xc = xfrm_state_clone(x, encap);
1524	1628	if (!xc)
1525	1629	return NULL;
	1630	+
	1631	+ xc->props.family = m->new_family;
	1632	+
	1633	+ if (xfrm_init_state(xc) < 0)
	1634	+ goto error;
1526	1635
1527	1636	memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
1528	1637	memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
..	..	@@ -1605,7 +1714,8 @@
1605	1714	memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
1606	1715	x1->km.dying = 0;
1607	1716
1608		- tasklet_hrtimer_start(&x1->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL);
	1717	+ hrtimer_start(&x1->mtimer, ktime_set(1, 0),
	1718	+ HRTIMER_MODE_REL_SOFT);
1609	1719	if (x1->curlft.use_time)
1610	1720	xfrm_state_check_expire(x1);
1611	1721
..	..	@@ -1644,7 +1754,7 @@
1644	1754	if (x->curlft.bytes >= x->lft.hard_byte_limit \|\|
1645	1755	x->curlft.packets >= x->lft.hard_packet_limit) {
1646	1756	x->km.state = XFRM_STATE_EXPIRED;
1647		- tasklet_hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL);
	1757	+ hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);
1648	1758	return -EINVAL;
1649	1759	}
1650	1760
..	..	@@ -1701,51 +1811,129 @@
1701	1811	EXPORT_SYMBOL(xfrm_find_acq);
1702	1812
1703	1813	#ifdef CONFIG_XFRM_SUB_POLICY
1704		-int
1705		-xfrm_tmpl_sort(struct xfrm_tmpl dst, struct xfrm_tmpl src, int n,
1706		- unsigned short family, struct net *net)
	1814	+#if IS_ENABLED(CONFIG_IPV6)
	1815	+/* distribution counting sort function for xfrm_state and xfrm_tmpl */
	1816	+static void
	1817	+__xfrm6_sort(void dst, void src, int n,
	1818	+ int (cmp)(const void p), int maxclass)
	1819	+{
	1820	+ int count[XFRM_MAX_DEPTH] = { };
	1821	+ int class[XFRM_MAX_DEPTH];
	1822	+ int i;
	1823	+
	1824	+ for (i = 0; i < n; i++) {
	1825	+ int c = cmp(src[i]);
	1826	+
	1827	+ class[i] = c;
	1828	+ count[c]++;
	1829	+ }
	1830	+
	1831	+ for (i = 2; i < maxclass; i++)
	1832	+ count[i] += count[i - 1];
	1833	+
	1834	+ for (i = 0; i < n; i++) {
	1835	+ dst[count[class[i] - 1]++] = src[i];
	1836	+ src[i] = NULL;
	1837	+ }
	1838	+}
	1839	+
	1840	+/* Rule for xfrm_state:
	1841	+ *
	1842	+ * rule 1: select IPsec transport except AH
	1843	+ * rule 2: select MIPv6 RO or inbound trigger
	1844	+ * rule 3: select IPsec transport AH
	1845	+ * rule 4: select IPsec tunnel
	1846	+ * rule 5: others
	1847	+ */
	1848	+static int __xfrm6_state_sort_cmp(const void *p)
	1849	+{
	1850	+ const struct xfrm_state *v = p;
	1851	+
	1852	+ switch (v->props.mode) {
	1853	+ case XFRM_MODE_TRANSPORT:
	1854	+ if (v->id.proto != IPPROTO_AH)
	1855	+ return 1;
	1856	+ else
	1857	+ return 3;
	1858	+#if IS_ENABLED(CONFIG_IPV6_MIP6)
	1859	+ case XFRM_MODE_ROUTEOPTIMIZATION:
	1860	+ case XFRM_MODE_IN_TRIGGER:
	1861	+ return 2;
	1862	+#endif
	1863	+ case XFRM_MODE_TUNNEL:
	1864	+ case XFRM_MODE_BEET:
	1865	+ return 4;
	1866	+ }
	1867	+ return 5;
	1868	+}
	1869	+
	1870	+/* Rule for xfrm_tmpl:
	1871	+ *
	1872	+ * rule 1: select IPsec transport
	1873	+ * rule 2: select MIPv6 RO or inbound trigger
	1874	+ * rule 3: select IPsec tunnel
	1875	+ * rule 4: others
	1876	+ */
	1877	+static int __xfrm6_tmpl_sort_cmp(const void *p)
	1878	+{
	1879	+ const struct xfrm_tmpl *v = p;
	1880	+
	1881	+ switch (v->mode) {
	1882	+ case XFRM_MODE_TRANSPORT:
	1883	+ return 1;
	1884	+#if IS_ENABLED(CONFIG_IPV6_MIP6)
	1885	+ case XFRM_MODE_ROUTEOPTIMIZATION:
	1886	+ case XFRM_MODE_IN_TRIGGER:
	1887	+ return 2;
	1888	+#endif
	1889	+ case XFRM_MODE_TUNNEL:
	1890	+ case XFRM_MODE_BEET:
	1891	+ return 3;
	1892	+ }
	1893	+ return 4;
	1894	+}
	1895	+#else
	1896	+static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
	1897	+static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
	1898	+
	1899	+static inline void
	1900	+__xfrm6_sort(void dst, void src, int n,
	1901	+ int (cmp)(const void p), int maxclass)
1707	1902	{
1708	1903	int i;
1709		- int err = 0;
1710		- struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
1711		- if (!afinfo)
1712		- return -EAFNOSUPPORT;
1713	1904
1714		- spin_lock_bh(&net->xfrm.xfrm_state_lock); /FIXME/
1715		- if (afinfo->tmpl_sort)
1716		- err = afinfo->tmpl_sort(dst, src, n);
	1905	+ for (i = 0; i < n; i++)
	1906	+ dst[i] = src[i];
	1907	+}
	1908	+#endif /* CONFIG_IPV6 */
	1909	+
	1910	+void
	1911	+xfrm_tmpl_sort(struct xfrm_tmpl dst, struct xfrm_tmpl src, int n,
	1912	+ unsigned short family)
	1913	+{
	1914	+ int i;
	1915	+
	1916	+ if (family == AF_INET6)
	1917	+ __xfrm6_sort((void )dst, (void )src, n,
	1918	+ __xfrm6_tmpl_sort_cmp, 5);
1717	1919	else
1718	1920	for (i = 0; i < n; i++)
1719	1921	dst[i] = src[i];
1720		- spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1721		- rcu_read_unlock();
1722		- return err;
1723	1922	}
1724		-EXPORT_SYMBOL(xfrm_tmpl_sort);
1725	1923
1726		-int
	1924	+void
1727	1925	xfrm_state_sort(struct xfrm_state dst, struct xfrm_state src, int n,
1728	1926	unsigned short family)
1729	1927	{
1730	1928	int i;
1731		- int err = 0;
1732		- struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
1733		- struct net net = xs_net(src);
1734	1929
1735		- if (!afinfo)
1736		- return -EAFNOSUPPORT;
1737		-
1738		- spin_lock_bh(&net->xfrm.xfrm_state_lock);
1739		- if (afinfo->state_sort)
1740		- err = afinfo->state_sort(dst, src, n);
	1930	+ if (family == AF_INET6)
	1931	+ __xfrm6_sort((void )dst, (void )src, n,
	1932	+ __xfrm6_state_sort_cmp, 6);
1741	1933	else
1742	1934	for (i = 0; i < n; i++)
1743	1935	dst[i] = src[i];
1744		- spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1745		- rcu_read_unlock();
1746		- return err;
1747	1936	}
1748		-EXPORT_SYMBOL(xfrm_state_sort);
1749	1937	#endif
1750	1938
1751	1939	/* Silly enough, but I'm lazy to build resolution list */
..	..	@@ -2102,7 +2290,7 @@
2102	2290	}
2103	2291	EXPORT_SYMBOL(km_report);
2104	2292
2105		-bool km_is_alive(const struct km_event *c)
	2293	+static bool km_is_alive(const struct km_event *c)
2106	2294	{
2107	2295	struct xfrm_mgr *km;
2108	2296	bool is_alive = false;
..	..	@@ -2118,7 +2306,6 @@
2118	2306
2119	2307	return is_alive;
2120	2308	}
2121		-EXPORT_SYMBOL(km_is_alive);
2122	2309
2123	2310	#if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2124	2311	static DEFINE_SPINLOCK(xfrm_translator_lock);
..	..	@@ -2180,14 +2367,14 @@
2180	2367	EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
2181	2368	#endif
2182	2369
2183		-int xfrm_user_policy(struct sock sk, int optname, u8 __user optval, int optlen)
	2370	+int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
2184	2371	{
2185	2372	int err;
2186	2373	u8 *data;
2187	2374	struct xfrm_mgr *km;
2188	2375	struct xfrm_policy *pol = NULL;
2189	2376
2190		- if (!optval && !optlen) {
	2377	+ if (sockptr_is_null(optval) && !optlen) {
2191	2378	xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
2192	2379	xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
2193	2380	__sk_dst_reset(sk);
..	..	@@ -2197,7 +2384,7 @@
2197	2384	if (optlen <= 0 \|\| optlen > PAGE_SIZE)
2198	2385	return -EMSGSIZE;
2199	2386
2200		- data = memdup_user(optval, optlen);
	2387	+ data = memdup_sockptr(optval, optlen);
2201	2388	if (IS_ERR(data))
2202	2389	return PTR_ERR(data);
2203	2390
..	..	@@ -2206,8 +2393,10 @@
2206	2393	if (in_compat_syscall()) {
2207	2394	struct xfrm_translator *xtr = xfrm_get_translator();
2208	2395
2209		- if (!xtr)
	2396	+ if (!xtr) {
	2397	+ kfree(data);
2210	2398	return -EOPNOTSUPP;
	2399	+ }
2211	2400
2212	2401	err = xtr->xlate_user_policy_sockptr(&data, optlen);
2213	2402	xfrm_put_translator(xtr);
..	..	@@ -2305,6 +2494,7 @@
2305	2494
2306	2495	return rcu_dereference(xfrm_state_afinfo[family]);
2307	2496	}
	2497	+EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
2308	2498
2309	2499	struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
2310	2500	{
..	..	@@ -2339,37 +2529,49 @@
2339	2529	}
2340	2530	EXPORT_SYMBOL(xfrm_state_delete_tunnel);
2341	2531
2342		-int xfrm_state_mtu(struct xfrm_state *x, int mtu)
	2532	+u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
2343	2533	{
2344	2534	const struct xfrm_type *type = READ_ONCE(x->type);
	2535	+ struct crypto_aead *aead;
	2536	+ u32 blksize, net_adj = 0;
2345	2537
2346		- if (x->km.state == XFRM_STATE_VALID &&
2347		- type && type->get_mtu)
2348		- return type->get_mtu(x, mtu);
	2538	+ if (x->km.state != XFRM_STATE_VALID \|\|
	2539	+ !type \|\| type->proto != IPPROTO_ESP)
	2540	+ return mtu - x->props.header_len;
2349	2541
2350		- return mtu - x->props.header_len;
	2542	+ aead = x->data;
	2543	+ blksize = ALIGN(crypto_aead_blocksize(aead), 4);
	2544	+
	2545	+ switch (x->props.mode) {
	2546	+ case XFRM_MODE_TRANSPORT:
	2547	+ case XFRM_MODE_BEET:
	2548	+ if (x->props.family == AF_INET)
	2549	+ net_adj = sizeof(struct iphdr);
	2550	+ else if (x->props.family == AF_INET6)
	2551	+ net_adj = sizeof(struct ipv6hdr);
	2552	+ break;
	2553	+ case XFRM_MODE_TUNNEL:
	2554	+ break;
	2555	+ default:
	2556	+ WARN_ON_ONCE(1);
	2557	+ break;
	2558	+ }
	2559	+
	2560	+ return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
	2561	+ net_adj) & ~(blksize - 1)) + net_adj - 2;
2351	2562	}
	2563	+EXPORT_SYMBOL_GPL(xfrm_state_mtu);
2352	2564
2353	2565	int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
2354	2566	{
2355		- struct xfrm_state_afinfo *afinfo;
2356		- struct xfrm_mode *inner_mode;
	2567	+ const struct xfrm_mode *inner_mode;
	2568	+ const struct xfrm_mode *outer_mode;
2357	2569	int family = x->props.family;
2358	2570	int err;
2359	2571
2360		- err = -EAFNOSUPPORT;
2361		- afinfo = xfrm_state_get_afinfo(family);
2362		- if (!afinfo)
2363		- goto error;
2364		-
2365		- err = 0;
2366		- if (afinfo->init_flags)
2367		- err = afinfo->init_flags(x);
2368		-
2369		- rcu_read_unlock();
2370		-
2371		- if (err)
2372		- goto error;
	2572	+ if (family == AF_INET &&
	2573	+ READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
	2574	+ x->props.flags \|= XFRM_STATE_NOPMTUDISC;
2373	2575
2374	2576	err = -EPROTONOSUPPORT;
2375	2577
..	..	@@ -2379,25 +2581,19 @@
2379	2581	goto error;
2380	2582
2381	2583	if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2382		- family != x->sel.family) {
2383		- xfrm_put_mode(inner_mode);
	2584	+ family != x->sel.family)
2384	2585	goto error;
2385		- }
2386	2586
2387		- x->inner_mode = inner_mode;
	2587	+ x->inner_mode = *inner_mode;
2388	2588	} else {
2389		- struct xfrm_mode *inner_mode_iaf;
	2589	+ const struct xfrm_mode *inner_mode_iaf;
2390	2590	int iafamily = AF_INET;
2391	2591
2392	2592	inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
2393	2593	if (inner_mode == NULL)
2394	2594	goto error;
2395	2595
2396		- if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) {
2397		- xfrm_put_mode(inner_mode);
2398		- goto error;
2399		- }
2400		- x->inner_mode = inner_mode;
	2596	+ x->inner_mode = *inner_mode;
2401	2597
2402	2598	if (x->props.family == AF_INET)
2403	2599	iafamily = AF_INET6;
..	..	@@ -2405,9 +2601,7 @@
2405	2601	inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
2406	2602	if (inner_mode_iaf) {
2407	2603	if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
2408		- x->inner_mode_iaf = inner_mode_iaf;
2409		- else
2410		- xfrm_put_mode(inner_mode_iaf);
	2604	+ x->inner_mode_iaf = *inner_mode_iaf;
2411	2605	}
2412	2606	}
2413	2607
..	..	@@ -2421,12 +2615,13 @@
2421	2615	if (err)
2422	2616	goto error;
2423	2617
2424		- x->outer_mode = xfrm_get_mode(x->props.mode, family);
2425		- if (x->outer_mode == NULL) {
	2618	+ outer_mode = xfrm_get_mode(x->props.mode, family);
	2619	+ if (!outer_mode) {
2426	2620	err = -EPROTONOSUPPORT;
2427	2621	goto error;
2428	2622	}
2429	2623
	2624	+ x->outer_mode = *outer_mode;
2430	2625	if (init_replay) {
2431	2626	err = xfrm_init_replay(x);
2432	2627	if (err)
..	..	@@ -2478,6 +2673,7 @@
2478	2673	net->xfrm.state_num = 0;
2479	2674	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
2480	2675	spin_lock_init(&net->xfrm.xfrm_state_lock);
	2676	+ seqcount_init(&net->xfrm.xfrm_state_hash_generation);
2481	2677	return 0;
2482	2678
2483	2679	out_byspi: