~ljy/RK356X_SDK_RELEASE.git

..	..	@@ -39,8 +39,11 @@
39	39	#include <linux/netdevice.h>
40	40	#include <linux/sched/signal.h>
41	41	#include <linux/inetdevice.h>
	42	+#include <linux/inet_diag.h>
42	43
	44	+#include <net/snmp.h>
43	45	#include <net/tls.h>
	46	+#include <net/tls_toe.h>
44	47
45	48	MODULE_AUTHOR("Mellanox Technologies");
46	49	MODULE_DESCRIPTION("Transport Layer Security Support");
..	..	@@ -53,18 +56,23 @@
53	56	TLS_NUM_PROTS,
54	57	};
55	58
56		-static struct proto *saved_tcpv6_prot;
	59	+static const struct proto *saved_tcpv6_prot;
57	60	static DEFINE_MUTEX(tcpv6_prot_mutex);
58		-static LIST_HEAD(device_list);
59		-static DEFINE_MUTEX(device_mutex);
	61	+static const struct proto *saved_tcpv4_prot;
	62	+static DEFINE_MUTEX(tcpv4_prot_mutex);
60	63	static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
61		-static struct proto_ops tls_sw_proto_ops;
	64	+static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
	65	+static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
	66	+ const struct proto *base);
62	67
63		-static void update_sk_prot(struct sock sk, struct tls_context ctx)
	68	+void update_sk_prot(struct sock sk, struct tls_context ctx)
64	69	{
65	70	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
66	71
67		- sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf];
	72	+ WRITE_ONCE(sk->sk_prot,
	73	+ &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]);
	74	+ WRITE_ONCE(sk->sk_socket->ops,
	75	+ &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]);
68	76	}
69	77
70	78	int wait_on_pending_writer(struct sock sk, long timeo)
..	..	@@ -141,9 +149,7 @@
141	149	size = sg->length;
142	150	}
143	151
144		- clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
145	152	ctx->in_tcp_sendpages = false;
146		- ctx->sk_write_space(sk);
147	153
148	154	return 0;
149	155	}
..	..	@@ -193,20 +199,28 @@
193	199	return rc;
194	200	}
195	201
196		-int tls_push_pending_closed_record(struct sock sk, struct tls_context ctx,
197		- int flags, long *timeo)
	202	+int tls_push_partial_record(struct sock sk, struct tls_context ctx,
	203	+ int flags)
198	204	{
199	205	struct scatterlist *sg;
200	206	u16 offset;
201		-
202		- if (!tls_is_partially_sent_record(ctx))
203		- return ctx->push_pending_record(sk, flags);
204	207
205	208	sg = ctx->partially_sent_record;
206	209	offset = ctx->partially_sent_offset;
207	210
208	211	ctx->partially_sent_record = NULL;
209	212	return tls_push_sg(sk, ctx, sg, offset, flags);
	213	+}
	214	+
	215	+void tls_free_partial_record(struct sock sk, struct tls_context ctx)
	216	+{
	217	+ struct scatterlist *sg;
	218	+
	219	+ for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) {
	220	+ put_page(sg_page(sg));
	221	+ sk_mem_uncharge(sk, sg->length);
	222	+ }
	223	+ ctx->partially_sent_record = NULL;
210	224	}
211	225
212	226	static void tls_write_space(struct sock *sk)
..	..	@@ -222,107 +236,109 @@
222	236	return;
223	237	}
224	238
225		- if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
226		- gfp_t sk_allocation = sk->sk_allocation;
227		- int rc;
228		- long timeo = 0;
229		-
230		- sk->sk_allocation = GFP_ATOMIC;
231		- rc = tls_push_pending_closed_record(sk, ctx,
232		- MSG_DONTWAIT \|
233		- MSG_NOSIGNAL,
234		- &timeo);
235		- sk->sk_allocation = sk_allocation;
236		-
237		- if (rc < 0)
238		- return;
239		- }
	239	+#ifdef CONFIG_TLS_DEVICE
	240	+ if (ctx->tx_conf == TLS_HW)
	241	+ tls_device_write_space(sk, ctx);
	242	+ else
	243	+#endif
	244	+ tls_sw_write_space(sk, ctx);
240	245
241	246	ctx->sk_write_space(sk);
242	247	}
243	248
244		-void tls_ctx_free(struct tls_context *ctx)
	249	+/**
	250	+ * tls_ctx_free() - free TLS ULP context
	251	+ * @sk: socket to with @ctx is attached
	252	+ * @ctx: TLS context structure
	253	+ *
	254	+ * Free TLS context. If @sk is %NULL caller guarantees that the socket
	255	+ * to which @ctx was attached has no outstanding references.
	256	+ */
	257	+void tls_ctx_free(struct sock sk, struct tls_context ctx)
245	258	{
246	259	if (!ctx)
247	260	return;
248	261
249	262	memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send));
250	263	memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv));
251		- kfree(ctx);
	264	+ mutex_destroy(&ctx->tx_lock);
	265	+
	266	+ if (sk)
	267	+ kfree_rcu(ctx, rcu);
	268	+ else
	269	+ kfree(ctx);
252	270	}
253	271
254		-static void tls_sk_proto_close(struct sock *sk, long timeout)
	272	+static void tls_sk_proto_cleanup(struct sock *sk,
	273	+ struct tls_context *ctx, long timeo)
255	274	{
256		- struct tls_context *ctx = tls_get_ctx(sk);
257		- long timeo = sock_sndtimeo(sk, 0);
258		- void (sk_proto_close)(struct sock sk, long timeout);
259		- bool free_ctx = false;
260		-
261		- lock_sock(sk);
262		- sk_proto_close = ctx->sk_proto_close;
263		-
264		- if ((ctx->tx_conf == TLS_HW_RECORD && ctx->rx_conf == TLS_HW_RECORD) \|\|
265		- (ctx->tx_conf == TLS_BASE && ctx->rx_conf == TLS_BASE)) {
266		- free_ctx = true;
267		- goto skip_tx_cleanup;
268		- }
269		-
270		- if (!tls_complete_pending_work(sk, ctx, 0, &timeo))
	275	+ if (unlikely(sk->sk_write_pending) &&
	276	+ !wait_on_pending_writer(sk, &timeo))
271	277	tls_handle_open_record(sk, 0);
272		-
273		- if (ctx->partially_sent_record) {
274		- struct scatterlist *sg = ctx->partially_sent_record;
275		-
276		- while (1) {
277		- put_page(sg_page(sg));
278		- sk_mem_uncharge(sk, sg->length);
279		-
280		- if (sg_is_last(sg))
281		- break;
282		- sg++;
283		- }
284		- }
285	278
286	279	/* We need these for tls_sw_fallback handling of other packets */
287	280	if (ctx->tx_conf == TLS_SW) {
288	281	kfree(ctx->tx.rec_seq);
289	282	kfree(ctx->tx.iv);
290		- tls_sw_free_resources_tx(sk);
	283	+ tls_sw_release_resources_tx(sk);
	284	+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
	285	+ } else if (ctx->tx_conf == TLS_HW) {
	286	+ tls_device_free_resources_tx(sk);
	287	+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
291	288	}
292	289
293		- if (ctx->rx_conf == TLS_SW)
294		- tls_sw_free_resources_rx(sk);
295		-
296		-#ifdef CONFIG_TLS_DEVICE
297		- if (ctx->rx_conf == TLS_HW)
	290	+ if (ctx->rx_conf == TLS_SW) {
	291	+ tls_sw_release_resources_rx(sk);
	292	+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
	293	+ } else if (ctx->rx_conf == TLS_HW) {
298	294	tls_device_offload_cleanup_rx(sk);
299		-
300		- if (ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW) {
301		-#else
302		- {
303		-#endif
304		- if (sk->sk_write_space == tls_write_space)
305		- sk->sk_write_space = ctx->sk_write_space;
306		- tls_ctx_free(ctx);
307		- ctx = NULL;
	295	+ TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
308	296	}
309		-
310		-skip_tx_cleanup:
311		- release_sock(sk);
312		- sk_proto_close(sk, timeout);
313		- /* free ctx for TLS_HW_RECORD, used by tcp_set_state
314		- * for sk->sk_prot->unhash [tls_hw_unhash]
315		- */
316		- if (free_ctx)
317		- tls_ctx_free(ctx);
318	297	}
319	298
320		-static int do_tls_getsockopt_tx(struct sock sk, char __user optval,
321		- int __user *optlen)
	299	+static void tls_sk_proto_close(struct sock *sk, long timeout)
	300	+{
	301	+ struct inet_connection_sock *icsk = inet_csk(sk);
	302	+ struct tls_context *ctx = tls_get_ctx(sk);
	303	+ long timeo = sock_sndtimeo(sk, 0);
	304	+ bool free_ctx;
	305	+
	306	+ if (ctx->tx_conf == TLS_SW)
	307	+ tls_sw_cancel_work_tx(ctx);
	308	+
	309	+ lock_sock(sk);
	310	+ free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
	311	+
	312	+ if (ctx->tx_conf != TLS_BASE \|\| ctx->rx_conf != TLS_BASE)
	313	+ tls_sk_proto_cleanup(sk, ctx, timeo);
	314	+
	315	+ write_lock_bh(&sk->sk_callback_lock);
	316	+ if (free_ctx)
	317	+ rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
	318	+ WRITE_ONCE(sk->sk_prot, ctx->sk_proto);
	319	+ if (sk->sk_write_space == tls_write_space)
	320	+ sk->sk_write_space = ctx->sk_write_space;
	321	+ write_unlock_bh(&sk->sk_callback_lock);
	322	+ release_sock(sk);
	323	+ if (ctx->tx_conf == TLS_SW)
	324	+ tls_sw_free_ctx_tx(ctx);
	325	+ if (ctx->rx_conf == TLS_SW \|\| ctx->rx_conf == TLS_HW)
	326	+ tls_sw_strparser_done(ctx);
	327	+ if (ctx->rx_conf == TLS_SW)
	328	+ tls_sw_free_ctx_rx(ctx);
	329	+ ctx->sk_proto->close(sk, timeout);
	330	+
	331	+ if (free_ctx)
	332	+ tls_ctx_free(sk, ctx);
	333	+}
	334	+
	335	+static int do_tls_getsockopt_conf(struct sock sk, char __user optval,
	336	+ int __user *optlen, int tx)
322	337	{
323	338	int rc = 0;
324	339	struct tls_context *ctx = tls_get_ctx(sk);
325	340	struct tls_crypto_info *crypto_info;
	341	+ struct cipher_context *cctx;
326	342	int len;
327	343
328	344	if (get_user(len, optlen))
..	..	@@ -339,7 +355,13 @@
339	355	}
340	356
341	357	/* get user crypto info */
342		- crypto_info = &ctx->crypto_send.info;
	358	+ if (tx) {
	359	+ crypto_info = &ctx->crypto_send.info;
	360	+ cctx = &ctx->tx;
	361	+ } else {
	362	+ crypto_info = &ctx->crypto_recv.info;
	363	+ cctx = &ctx->rx;
	364	+ }
343	365
344	366	if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
345	367	rc = -EBUSY;
..	..	@@ -366,14 +388,38 @@
366	388	}
367	389	lock_sock(sk);
368	390	memcpy(crypto_info_aes_gcm_128->iv,
369		- ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
	391	+ cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
370	392	TLS_CIPHER_AES_GCM_128_IV_SIZE);
371		- memcpy(crypto_info_aes_gcm_128->rec_seq, ctx->tx.rec_seq,
	393	+ memcpy(crypto_info_aes_gcm_128->rec_seq, cctx->rec_seq,
372	394	TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
373	395	release_sock(sk);
374	396	if (copy_to_user(optval,
375	397	crypto_info_aes_gcm_128,
376	398	sizeof(*crypto_info_aes_gcm_128)))
	399	+ rc = -EFAULT;
	400	+ break;
	401	+ }
	402	+ case TLS_CIPHER_AES_GCM_256: {
	403	+ struct tls12_crypto_info_aes_gcm_256 *
	404	+ crypto_info_aes_gcm_256 =
	405	+ container_of(crypto_info,
	406	+ struct tls12_crypto_info_aes_gcm_256,
	407	+ info);
	408	+
	409	+ if (len != sizeof(*crypto_info_aes_gcm_256)) {
	410	+ rc = -EINVAL;
	411	+ goto out;
	412	+ }
	413	+ lock_sock(sk);
	414	+ memcpy(crypto_info_aes_gcm_256->iv,
	415	+ cctx->iv + TLS_CIPHER_AES_GCM_256_SALT_SIZE,
	416	+ TLS_CIPHER_AES_GCM_256_IV_SIZE);
	417	+ memcpy(crypto_info_aes_gcm_256->rec_seq, cctx->rec_seq,
	418	+ TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE);
	419	+ release_sock(sk);
	420	+ if (copy_to_user(optval,
	421	+ crypto_info_aes_gcm_256,
	422	+ sizeof(*crypto_info_aes_gcm_256)))
377	423	rc = -EFAULT;
378	424	break;
379	425	}
..	..	@@ -392,7 +438,9 @@
392	438
393	439	switch (optname) {
394	440	case TLS_TX:
395		- rc = do_tls_getsockopt_tx(sk, optval, optlen);
	441	+ case TLS_RX:
	442	+ rc = do_tls_getsockopt_conf(sk, optval, optlen,
	443	+ optname == TLS_TX);
396	444	break;
397	445	default:
398	446	rc = -ENOPROTOOPT;
..	..	@@ -407,28 +455,34 @@
407	455	struct tls_context *ctx = tls_get_ctx(sk);
408	456
409	457	if (level != SOL_TLS)
410		- return ctx->getsockopt(sk, level, optname, optval, optlen);
	458	+ return ctx->sk_proto->getsockopt(sk, level,
	459	+ optname, optval, optlen);
411	460
412	461	return do_tls_getsockopt(sk, optname, optval, optlen);
413	462	}
414	463
415		-static int do_tls_setsockopt_conf(struct sock sk, char __user optval,
	464	+static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
416	465	unsigned int optlen, int tx)
417	466	{
418	467	struct tls_crypto_info *crypto_info;
	468	+ struct tls_crypto_info *alt_crypto_info;
419	469	struct tls_context *ctx = tls_get_ctx(sk);
	470	+ size_t optsize;
420	471	int rc = 0;
421	472	int conf;
422	473
423		- if (!optval \|\| (optlen < sizeof(*crypto_info))) {
	474	+ if (sockptr_is_null(optval) \|\| (optlen < sizeof(*crypto_info))) {
424	475	rc = -EINVAL;
425	476	goto out;
426	477	}
427	478
428		- if (tx)
	479	+ if (tx) {
429	480	crypto_info = &ctx->crypto_send.info;
430		- else
	481	+ alt_crypto_info = &ctx->crypto_recv.info;
	482	+ } else {
431	483	crypto_info = &ctx->crypto_recv.info;
	484	+ alt_crypto_info = &ctx->crypto_send.info;
	485	+ }
432	486
433	487	/* Currently we don't support set crypto info more than one time */
434	488	if (TLS_CRYPTO_INFO_READY(crypto_info)) {
..	..	@@ -436,63 +490,87 @@
436	490	goto out;
437	491	}
438	492
439		- rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info));
	493	+ rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info));
440	494	if (rc) {
441	495	rc = -EFAULT;
442	496	goto err_crypto_info;
443	497	}
444	498
445	499	/* check version */
446		- if (crypto_info->version != TLS_1_2_VERSION) {
447		- rc = -ENOTSUPP;
	500	+ if (crypto_info->version != TLS_1_2_VERSION &&
	501	+ crypto_info->version != TLS_1_3_VERSION) {
	502	+ rc = -EINVAL;
448	503	goto err_crypto_info;
449	504	}
450	505
451		- switch (crypto_info->cipher_type) {
452		- case TLS_CIPHER_AES_GCM_128: {
453		- if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) {
	506	+ /* Ensure that TLS version and ciphers are same in both directions */
	507	+ if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) {
	508	+ if (alt_crypto_info->version != crypto_info->version \|\|
	509	+ alt_crypto_info->cipher_type != crypto_info->cipher_type) {
454	510	rc = -EINVAL;
455	511	goto err_crypto_info;
456	512	}
457		- rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info),
458		- optlen - sizeof(*crypto_info));
459		- if (rc) {
460		- rc = -EFAULT;
461		- goto err_crypto_info;
462		- }
	513	+ }
	514	+
	515	+ switch (crypto_info->cipher_type) {
	516	+ case TLS_CIPHER_AES_GCM_128:
	517	+ optsize = sizeof(struct tls12_crypto_info_aes_gcm_128);
	518	+ break;
	519	+ case TLS_CIPHER_AES_GCM_256: {
	520	+ optsize = sizeof(struct tls12_crypto_info_aes_gcm_256);
463	521	break;
464	522	}
	523	+ case TLS_CIPHER_AES_CCM_128:
	524	+ optsize = sizeof(struct tls12_crypto_info_aes_ccm_128);
	525	+ break;
465	526	default:
466	527	rc = -EINVAL;
467	528	goto err_crypto_info;
468	529	}
469	530
	531	+ if (optlen != optsize) {
	532	+ rc = -EINVAL;
	533	+ goto err_crypto_info;
	534	+ }
	535	+
	536	+ rc = copy_from_sockptr_offset(crypto_info + 1, optval,
	537	+ sizeof(*crypto_info),
	538	+ optlen - sizeof(*crypto_info));
	539	+ if (rc) {
	540	+ rc = -EFAULT;
	541	+ goto err_crypto_info;
	542	+ }
	543	+
470	544	if (tx) {
471		-#ifdef CONFIG_TLS_DEVICE
472	545	rc = tls_set_device_offload(sk, ctx);
473	546	conf = TLS_HW;
474		- if (rc) {
475		-#else
476		- {
477		-#endif
	547	+ if (!rc) {
	548	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
	549	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
	550	+ } else {
478	551	rc = tls_set_sw_offload(sk, ctx, 1);
	552	+ if (rc)
	553	+ goto err_crypto_info;
	554	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
	555	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
479	556	conf = TLS_SW;
480	557	}
481	558	} else {
482		-#ifdef CONFIG_TLS_DEVICE
483	559	rc = tls_set_device_offload_rx(sk, ctx);
484	560	conf = TLS_HW;
485		- if (rc) {
486		-#else
487		- {
488		-#endif
	561	+ if (!rc) {
	562	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
	563	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
	564	+ } else {
489	565	rc = tls_set_sw_offload(sk, ctx, 0);
	566	+ if (rc)
	567	+ goto err_crypto_info;
	568	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
	569	+ TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
490	570	conf = TLS_SW;
491	571	}
	572	+ tls_sw_strparser_arm(sk, ctx);
492	573	}
493		-
494		- if (rc)
495		- goto err_crypto_info;
496	574
497	575	if (tx)
498	576	ctx->tx_conf = conf;
..	..	@@ -502,8 +580,6 @@
502	580	if (tx) {
503	581	ctx->sk_write_space = sk->sk_write_space;
504	582	sk->sk_write_space = tls_write_space;
505		- } else {
506		- sk->sk_socket->ops = &tls_sw_proto_ops;
507	583	}
508	584	goto out;
509	585
..	..	@@ -513,8 +589,8 @@
513	589	return rc;
514	590	}
515	591
516		-static int do_tls_setsockopt(struct sock *sk, int optname,
517		- char __user *optval, unsigned int optlen)
	592	+static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval,
	593	+ unsigned int optlen)
518	594	{
519	595	int rc = 0;
520	596
..	..	@@ -534,17 +610,18 @@
534	610	}
535	611
536	612	static int tls_setsockopt(struct sock *sk, int level, int optname,
537		- char __user *optval, unsigned int optlen)
	613	+ sockptr_t optval, unsigned int optlen)
538	614	{
539	615	struct tls_context *ctx = tls_get_ctx(sk);
540	616
541	617	if (level != SOL_TLS)
542		- return ctx->setsockopt(sk, level, optname, optval, optlen);
	618	+ return ctx->sk_proto->setsockopt(sk, level, optname, optval,
	619	+ optlen);
543	620
544	621	return do_tls_setsockopt(sk, optname, optval, optlen);
545	622	}
546	623
547		-static struct tls_context create_ctx(struct sock sk)
	624	+struct tls_context tls_ctx_create(struct sock sk)
548	625	{
549	626	struct inet_connection_sock *icsk = inet_csk(sk);
550	627	struct tls_context *ctx;
..	..	@@ -553,76 +630,79 @@
553	630	if (!ctx)
554	631	return NULL;
555	632
556		- icsk->icsk_ulp_data = ctx;
557		- ctx->setsockopt = sk->sk_prot->setsockopt;
558		- ctx->getsockopt = sk->sk_prot->getsockopt;
559		- ctx->sk_proto_close = sk->sk_prot->close;
	633	+ mutex_init(&ctx->tx_lock);
	634	+ rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
	635	+ ctx->sk_proto = READ_ONCE(sk->sk_prot);
	636	+ ctx->sk = sk;
560	637	return ctx;
561	638	}
562	639
563		-static int tls_hw_prot(struct sock *sk)
	640	+static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
	641	+ const struct proto_ops *base)
564	642	{
565		- struct tls_context *ctx;
566		- struct tls_device *dev;
567		- int rc = 0;
	643	+ ops[TLS_BASE][TLS_BASE] = *base;
568	644
569		- mutex_lock(&device_mutex);
570		- list_for_each_entry(dev, &device_list, dev_list) {
571		- if (dev->feature && dev->feature(dev)) {
572		- ctx = create_ctx(sk);
573		- if (!ctx)
574		- goto out;
	645	+ ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
	646	+ ops[TLS_SW ][TLS_BASE].sendpage_locked = tls_sw_sendpage_locked;
575	647
576		- ctx->hash = sk->sk_prot->hash;
577		- ctx->unhash = sk->sk_prot->unhash;
578		- ctx->sk_proto_close = sk->sk_prot->close;
579		- ctx->rx_conf = TLS_HW_RECORD;
580		- ctx->tx_conf = TLS_HW_RECORD;
581		- update_sk_prot(sk, ctx);
582		- rc = 1;
583		- break;
	648	+ ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE];
	649	+ ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read;
	650	+
	651	+ ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE];
	652	+ ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read;
	653	+
	654	+#ifdef CONFIG_TLS_DEVICE
	655	+ ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
	656	+ ops[TLS_HW ][TLS_BASE].sendpage_locked = NULL;
	657	+
	658	+ ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ];
	659	+ ops[TLS_HW ][TLS_SW ].sendpage_locked = NULL;
	660	+
	661	+ ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ];
	662	+
	663	+ ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ];
	664	+
	665	+ ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ];
	666	+ ops[TLS_HW ][TLS_HW ].sendpage_locked = NULL;
	667	+#endif
	668	+#ifdef CONFIG_TLS_TOE
	669	+ ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
	670	+#endif
	671	+}
	672	+
	673	+static void tls_build_proto(struct sock *sk)
	674	+{
	675	+ int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
	676	+ struct proto *prot = READ_ONCE(sk->sk_prot);
	677	+
	678	+ /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
	679	+ if (ip_ver == TLSV6 &&
	680	+ unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) {
	681	+ mutex_lock(&tcpv6_prot_mutex);
	682	+ if (likely(prot != saved_tcpv6_prot)) {
	683	+ build_protos(tls_prots[TLSV6], prot);
	684	+ build_proto_ops(tls_proto_ops[TLSV6],
	685	+ sk->sk_socket->ops);
	686	+ smp_store_release(&saved_tcpv6_prot, prot);
584	687	}
	688	+ mutex_unlock(&tcpv6_prot_mutex);
585	689	}
586		-out:
587		- mutex_unlock(&device_mutex);
588		- return rc;
589		-}
590	690
591		-static void tls_hw_unhash(struct sock *sk)
592		-{
593		- struct tls_context *ctx = tls_get_ctx(sk);
594		- struct tls_device *dev;
595		-
596		- mutex_lock(&device_mutex);
597		- list_for_each_entry(dev, &device_list, dev_list) {
598		- if (dev->unhash)
599		- dev->unhash(dev, sk);
	691	+ if (ip_ver == TLSV4 &&
	692	+ unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) {
	693	+ mutex_lock(&tcpv4_prot_mutex);
	694	+ if (likely(prot != saved_tcpv4_prot)) {
	695	+ build_protos(tls_prots[TLSV4], prot);
	696	+ build_proto_ops(tls_proto_ops[TLSV4],
	697	+ sk->sk_socket->ops);
	698	+ smp_store_release(&saved_tcpv4_prot, prot);
	699	+ }
	700	+ mutex_unlock(&tcpv4_prot_mutex);
600	701	}
601		- mutex_unlock(&device_mutex);
602		- ctx->unhash(sk);
603		-}
604		-
605		-static int tls_hw_hash(struct sock *sk)
606		-{
607		- struct tls_context *ctx = tls_get_ctx(sk);
608		- struct tls_device *dev;
609		- int err;
610		-
611		- err = ctx->hash(sk);
612		- mutex_lock(&device_mutex);
613		- list_for_each_entry(dev, &device_list, dev_list) {
614		- if (dev->hash)
615		- err \|= dev->hash(dev, sk);
616		- }
617		- mutex_unlock(&device_mutex);
618		-
619		- if (err)
620		- tls_hw_unhash(sk);
621		- return err;
622	702	}
623	703
624	704	static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
625		- struct proto *base)
	705	+ const struct proto *base)
626	706	{
627	707	prot[TLS_BASE][TLS_BASE] = *base;
628	708	prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt;
..	..	@@ -634,12 +714,14 @@
634	714	prot[TLS_SW][TLS_BASE].sendpage = tls_sw_sendpage;
635	715
636	716	prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
637		- prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg;
638		- prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close;
	717	+ prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg;
	718	+ prot[TLS_BASE][TLS_SW].stream_memory_read = tls_sw_stream_read;
	719	+ prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close;
639	720
640	721	prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
641		- prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg;
642		- prot[TLS_SW][TLS_SW].close = tls_sk_proto_close;
	722	+ prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg;
	723	+ prot[TLS_SW][TLS_SW].stream_memory_read = tls_sw_stream_read;
	724	+ prot[TLS_SW][TLS_SW].close = tls_sk_proto_close;
643	725
644	726	#ifdef CONFIG_TLS_DEVICE
645	727	prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
..	..	@@ -656,21 +738,24 @@
656	738
657	739	prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
658	740	#endif
659		-
	741	+#ifdef CONFIG_TLS_TOE
660	742	prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
661		- prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_hw_hash;
662		- prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_hw_unhash;
663		- prot[TLS_HW_RECORD][TLS_HW_RECORD].close = tls_sk_proto_close;
	743	+ prot[TLS_HW_RECORD][TLS_HW_RECORD].hash = tls_toe_hash;
	744	+ prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash = tls_toe_unhash;
	745	+#endif
664	746	}
665	747
666	748	static int tls_init(struct sock *sk)
667	749	{
668		- int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
669	750	struct tls_context *ctx;
670	751	int rc = 0;
671	752
672		- if (tls_hw_prot(sk))
673		- goto out;
	753	+ tls_build_proto(sk);
	754	+
	755	+#ifdef CONFIG_TLS_TOE
	756	+ if (tls_toe_bypass(sk))
	757	+ return 0;
	758	+#endif
674	759
675	760	/* The TLS ulp is currently supported only for TCP sockets
676	761	* in ESTABLISHED state.
..	..	@@ -679,68 +764,153 @@
679	764	* share the ulp context.
680	765	*/
681	766	if (sk->sk_state != TCP_ESTABLISHED)
682		- return -ENOTSUPP;
	767	+ return -ENOTCONN;
683	768
684	769	/* allocate tls context */
685		- ctx = create_ctx(sk);
	770	+ write_lock_bh(&sk->sk_callback_lock);
	771	+ ctx = tls_ctx_create(sk);
686	772	if (!ctx) {
687	773	rc = -ENOMEM;
688	774	goto out;
689		- }
690		-
691		- /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
692		- if (ip_ver == TLSV6 &&
693		- unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) {
694		- mutex_lock(&tcpv6_prot_mutex);
695		- if (likely(sk->sk_prot != saved_tcpv6_prot)) {
696		- build_protos(tls_prots[TLSV6], sk->sk_prot);
697		- smp_store_release(&saved_tcpv6_prot, sk->sk_prot);
698		- }
699		- mutex_unlock(&tcpv6_prot_mutex);
700	775	}
701	776
702	777	ctx->tx_conf = TLS_BASE;
703	778	ctx->rx_conf = TLS_BASE;
704	779	update_sk_prot(sk, ctx);
705	780	out:
	781	+ write_unlock_bh(&sk->sk_callback_lock);
706	782	return rc;
707	783	}
708	784
709		-void tls_register_device(struct tls_device *device)
	785	+static void tls_update(struct sock sk, struct proto p,
	786	+ void (write_space)(struct sock sk))
710	787	{
711		- mutex_lock(&device_mutex);
712		- list_add_tail(&device->dev_list, &device_list);
713		- mutex_unlock(&device_mutex);
714		-}
715		-EXPORT_SYMBOL(tls_register_device);
	788	+ struct tls_context *ctx;
716	789
717		-void tls_unregister_device(struct tls_device *device)
718		-{
719		- mutex_lock(&device_mutex);
720		- list_del(&device->dev_list);
721		- mutex_unlock(&device_mutex);
	790	+ ctx = tls_get_ctx(sk);
	791	+ if (likely(ctx)) {
	792	+ ctx->sk_write_space = write_space;
	793	+ ctx->sk_proto = p;
	794	+ } else {
	795	+ /* Pairs with lockless read in sk_clone_lock(). */
	796	+ WRITE_ONCE(sk->sk_prot, p);
	797	+ sk->sk_write_space = write_space;
	798	+ }
722	799	}
723		-EXPORT_SYMBOL(tls_unregister_device);
	800	+
	801	+static int tls_get_info(const struct sock sk, struct sk_buff skb)
	802	+{
	803	+ u16 version, cipher_type;
	804	+ struct tls_context *ctx;
	805	+ struct nlattr *start;
	806	+ int err;
	807	+
	808	+ start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS);
	809	+ if (!start)
	810	+ return -EMSGSIZE;
	811	+
	812	+ rcu_read_lock();
	813	+ ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data);
	814	+ if (!ctx) {
	815	+ err = 0;
	816	+ goto nla_failure;
	817	+ }
	818	+ version = ctx->prot_info.version;
	819	+ if (version) {
	820	+ err = nla_put_u16(skb, TLS_INFO_VERSION, version);
	821	+ if (err)
	822	+ goto nla_failure;
	823	+ }
	824	+ cipher_type = ctx->prot_info.cipher_type;
	825	+ if (cipher_type) {
	826	+ err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type);
	827	+ if (err)
	828	+ goto nla_failure;
	829	+ }
	830	+ err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true));
	831	+ if (err)
	832	+ goto nla_failure;
	833	+
	834	+ err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false));
	835	+ if (err)
	836	+ goto nla_failure;
	837	+
	838	+ rcu_read_unlock();
	839	+ nla_nest_end(skb, start);
	840	+ return 0;
	841	+
	842	+nla_failure:
	843	+ rcu_read_unlock();
	844	+ nla_nest_cancel(skb, start);
	845	+ return err;
	846	+}
	847	+
	848	+static size_t tls_get_info_size(const struct sock *sk)
	849	+{
	850	+ size_t size = 0;
	851	+
	852	+ size += nla_total_size(0) + /* INET_ULP_INFO_TLS */
	853	+ nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */
	854	+ nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */
	855	+ nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */
	856	+ nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */
	857	+ 0;
	858	+
	859	+ return size;
	860	+}
	861	+
	862	+static int __net_init tls_init_net(struct net *net)
	863	+{
	864	+ int err;
	865	+
	866	+ net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib);
	867	+ if (!net->mib.tls_statistics)
	868	+ return -ENOMEM;
	869	+
	870	+ err = tls_proc_init(net);
	871	+ if (err)
	872	+ goto err_free_stats;
	873	+
	874	+ return 0;
	875	+err_free_stats:
	876	+ free_percpu(net->mib.tls_statistics);
	877	+ return err;
	878	+}
	879	+
	880	+static void __net_exit tls_exit_net(struct net *net)
	881	+{
	882	+ tls_proc_fini(net);
	883	+ free_percpu(net->mib.tls_statistics);
	884	+}
	885	+
	886	+static struct pernet_operations tls_proc_ops = {
	887	+ .init = tls_init_net,
	888	+ .exit = tls_exit_net,
	889	+};
724	890
725	891	static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
726	892	.name = "tls",
727		- .uid = TCP_ULP_TLS,
728		- .user_visible = true,
729	893	.owner = THIS_MODULE,
730	894	.init = tls_init,
	895	+ .update = tls_update,
	896	+ .get_info = tls_get_info,
	897	+ .get_info_size = tls_get_info_size,
731	898	};
732	899
733	900	static int __init tls_register(void)
734	901	{
735		- build_protos(tls_prots[TLSV4], &tcp_prot);
	902	+ int err;
736	903
737		- tls_sw_proto_ops = inet_stream_ops;
738		- tls_sw_proto_ops.poll = tls_sw_poll;
739		- tls_sw_proto_ops.splice_read = tls_sw_splice_read;
	904	+ err = register_pernet_subsys(&tls_proc_ops);
	905	+ if (err)
	906	+ return err;
740	907
741		-#ifdef CONFIG_TLS_DEVICE
742		- tls_device_init();
743		-#endif
	908	+ err = tls_device_init();
	909	+ if (err) {
	910	+ unregister_pernet_subsys(&tls_proc_ops);
	911	+ return err;
	912	+ }
	913	+
744	914	tcp_register_ulp(&tcp_tls_ulp_ops);
745	915
746	916	return 0;
..	..	@@ -749,9 +919,8 @@
749	919	static void __exit tls_unregister(void)
750	920	{
751	921	tcp_unregister_ulp(&tcp_tls_ulp_ops);
752		-#ifdef CONFIG_TLS_DEVICE
753	922	tls_device_cleanup();
754		-#endif
	923	+ unregister_pernet_subsys(&tls_proc_ops);
755	924	}
756	925
757	926	module_init(tls_register);